ocxl_afp3_latency: Add new test to measure latencies

Import of internal test.

Signed-off-by: Frederic Barrat <fbarrat@linux.ibm.com>

5 files changed, 600 insertions, 25 deletions

diff --git a/Makefile b/Makefile
index e11bfd1..bf55f53 100644
--- a/Makefile
+++ b/Makefile
@@ -12,7 +12,7 @@ SONAMEOPT = -Wl,-soname,$(LIBSONAME)
 
 DOCDIR = docs
 
-all: check_ocxl_header obj/$(LIBSONAME) obj/libocxl.so obj/libocxl.a sampleobj/memcpy afuobj/ocxl_memcpy afuobj/ocxl_afp3
+all: check_ocxl_header obj/$(LIBSONAME) obj/libocxl.so obj/libocxl.a sampleobj/memcpy afuobj/ocxl_memcpy afuobj/ocxl_afp3 afuobj/ocxl_afp3_latency
 
 HAS_WGET = $(shell /bin/which wget > /dev/null 2>&1 && echo y || echo n)
 HAS_CURL = $(shell /bin/which curl > /dev/null 2>&1 && echo y || echo n)
@@ -62,6 +62,9 @@ afuobj/ocxl_memcpy: afuobj/ocxl_memcpy.o-memcpy
 afuobj/ocxl_afp3: afuobj/ocxl_afp3.o-afp
 	$(call Q,CC, $(CC) $(CFLAGS) $(LDFLAGS) -o afuobj/ocxl_afp3 afuobj/ocxl_afp3.o-afp obj/libocxl.a, afuobj/ocxl_afp3)
 
+afuobj/ocxl_afp3_latency: afuobj/ocxl_afp3_latency.o-afp
+	$(call Q,CC, $(CC) $(CFLAGS) $(LDFLAGS) -o afuobj/ocxl_afp3_latency afuobj/ocxl_afp3_latency.o-afp obj/libocxl.a, afuobj/ocxl_afp3_latency)
+
 testobj:
 	mkdir testobj
 
diff --git a/afutests/afp/README.md b/afutests/afp/README.md
index 86f9a71..3f25b4b 100644
--- a/afutests/afp/README.md
+++ b/afutests/afp/README.md
@@ -1,7 +1,7 @@
-ocxl_afp3
-=========
+ocxl_afp3, ocxl_afp3_latency
+============================
 
-`ocxl_afp3` is a test program for the OpenCAPI AFU IBM,AFP3 (used for
+`ocxl_afp3` and 'ocxl_afp3_latency' are test programs for the OpenCAPI AFU IBM,AFP3 (used for
 development).
 
 Requirements
@@ -40,6 +40,21 @@ Usage
                       Default 512kB.  Valid Range: 4K-512M.
                       Format: NumberLetter, e.g. 4K, 512K, 1M, 512M
         --timeout     Default=1 seconds
+	--device      Device to open instead of first AFP AFU found
         --verbose     Verbose output
         --help        Print this message
 ```
+
+
+    $ ../../afuobj/ocxl_afp3_latency
+    
+```
+    Usage: /home/fbarrat/capi/libocxl/afuobj/ocxl_afp3_latency [ options ]
+        -i 10000     --iterations       Default=10000
+        -p 64 to 512 --pong             Pong size from card to host (Bytes) Default=64
+        -x           --extraread        Add an DMA extraread before the DMA Wr Default is no
+        -f           --forever          Run until CTRL+C, Default=no
+        -d           --device           Device to open instead of first AFP AFU found
+        -v           --verbose          Verbose output
+        -h           --help             Print this message
+```
diff --git a/afutests/afp/ocxl_afp3.c b/afutests/afp/ocxl_afp3.c
index c75c43b..b86beaa 100644
--- a/afutests/afp/ocxl_afp3.c
+++ b/afutests/afp/ocxl_afp3.c
@@ -23,6 +23,8 @@
 #include <time.h>
 
 #include "libocxl.h"
+#include "ocxl_afp3.h"
+
 #define AFU_NAME "IBM,AFP3"
 
 #define CACHELINE_BYTES 128
@@ -31,27 +33,6 @@
 #define BUF_512MB 536870912
 #define BUF_4MB 4194304
 
-// global mmio registers
-#define AFUConfiguration_REGISTER     0x0000
-#define AFUInternalError_REGISTER     0x0010
-#define AFUInternalErrorInfo_REGISTER 0x0018
-#define AFUTraceControl_REGISTER      0x0020
-
-// global AFP3 registers
-#define AFUWED_AFP_REGISTER     0x0040
-#define AFUBufmask_AFP_REGISTER 0x0048
-#define AFUPASID_AFP_REGISTER   0x0050
-#define AFUEnable_AFP_REGISTER  0x0060
-#define AFUControl_AFP_REGISTER 0x0068
-#define AFUPerfCnt0_AFP_REGISTER 0x00C0
-#define AFUPerfCnt1_AFP_REGISTER 0x00C8
-#define AFUPerfCnt2_AFP_REGISTER 0x00D0
-#define AFUPerfCnt3_AFP_REGISTER 0x00D8
-#define AFUPerfCnt4_AFP_REGISTER 0x00E0
-#define AFUPerfCnt5_AFP_REGISTER 0x00E8
-#define AFUPerfCnt6_AFP_REGISTER 0x00F0
-#define AFUPerfCnt7_AFP_REGISTER 0x00F8
-
 
 static int verbose;
 static int timeout = 1;
diff --git a/afutests/afp/ocxl_afp3.h b/afutests/afp/ocxl_afp3.h
new file mode 100644
index 0000000..05f3f4d
--- /dev/null
+++ b/afutests/afp/ocxl_afp3.h
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2018 International Business Machines
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _OCXL_AFP3_H
+#define _OCXL_AFP3_H
+
+// global mmio registers
+#define AFUConfiguration_REGISTER     0x0000
+#define AFUInternalError_REGISTER     0x0010
+#define AFUInternalErrorInfo_REGISTER 0x0018
+#define AFUTraceControl_REGISTER      0x0020
+
+// global AFP3 registers
+#define AFUExtraReadEA_AFP_REGISTER  0x0038
+#define AFUWED_AFP_REGISTER          0x0040
+#define AFUBufmask_AFP_REGISTER      0x0048
+#define AFUPASID_AFP_REGISTER        0x0050
+#define AFUMisc_AFP_REGISTER         0x0058
+#define AFUEnable_AFP_REGISTER       0x0060
+#define AFUControl_AFP_REGISTER      0x0068
+#define AFULatency_AFP_REGISTER      0x0070
+#define AFUPerfCnt0_AFP_REGISTER     0x00C0
+#define AFUPerfCnt1_AFP_REGISTER     0x00C8
+#define AFUPerfCnt2_AFP_REGISTER     0x00D0
+#define AFUPerfCnt3_AFP_REGISTER     0x00D8
+#define AFUPerfCnt4_AFP_REGISTER     0x00E0
+#define AFUPerfCnt5_AFP_REGISTER     0x00E8
+#define AFUPerfCnt6_AFP_REGISTER     0x00F0
+#define AFUPerfCnt7_AFP_REGISTER     0x00F8
+#define Large_Data0_AFP_REGISTER     0x10000
+#define Large_Data1_AFP_REGISTER     0x10080
+#define Large_Data2_AFP_REGISTER     0x10100
+#define Large_Data3_AFP_REGISTER     0x10180
+
+#endif /* _OCXL_AFP3_H */
diff --git a/afutests/afp/ocxl_afp3_latency.c b/afutests/afp/ocxl_afp3_latency.c
new file mode 100644
index 0000000..b64c15c
--- /dev/null
+++ b/afutests/afp/ocxl_afp3_latency.c
@@ -0,0 +1,528 @@
+/*
+ * Copyright 2018 International Business Machines
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <getopt.h>
+#include <sys/mman.h>
+#include <signal.h>
+
+#include "libocxl.h"
+#include "ocxl_afp3.h"
+
+#define AFU_NAME "IBM,AFP3"
+#define BUF_512MB (512 * 1024 * 1024)
+#define BIT(n) (1ull << n)
+
+static char *device = NULL;
+static int verbose = 0;
+static int size_ping = 8;
+static int size_ld = 64;
+static int size_st = 64;
+static int extra_read = 0;
+static uint64_t iterations = 10000;
+
+static uint64_t disableAfu  = 0x0000000000000000;
+static uint64_t resetCnt    = 0x4000000000000000;
+
+#define miso()          asm volatile("or 26, 26, 26")
+
+#define mfspr(rn)       ({unsigned long rval;				\
+                        asm volatile("mfspr %0, %1"			\
+				: "=r" (rval) : "i" (rn)); rval;})
+
+static uint64_t read_timebase(void)
+{
+	return mfspr(268);
+}
+
+static void printf_buf(uint64_t addr, uint64_t size)
+{
+	unsigned int i, j;
+	uint8_t *base_p = (uint8_t *)addr;
+	uint64_t per_line = 32;
+
+	for (i = 0; i < size/per_line; i++) {
+		printf("0x%016lx:", (uint64_t) base_p);
+		for (j = 0; j < per_line; j++) {
+			if (j % 8 == 0)
+				printf(" ");
+
+			printf("%02x", *base_p);
+			base_p++;
+		}
+		printf("\n");
+	}
+	printf("\n");
+}
+
+static int flag_stop = 0;
+static void stop_handler(int signal)
+{
+	printf("Signal %d received, stopping\n", signal);
+	flag_stop = 1;
+}
+
+static inline uint64_t ping_8B(uint64_t global_mmio_start,
+			       volatile uint64_t *flag, uint64_t enable_in,
+			       uint64_t *count)
+{
+	uint64_t *afu_enable_reg_p;
+	uint64_t timebase[2];
+	uint64_t j, loop_count;
+
+	afu_enable_reg_p = (uint64_t *)(global_mmio_start +
+					AFUEnable_AFP_REGISTER);
+
+	if (*count) {
+		loop_count = *count;
+	} else {
+		loop_count = ~0ull;
+		printf("Running test forever, interrupt with ctrl-c\n");
+	}
+
+	timebase[0] = read_timebase();
+	for (j = 0; j < loop_count; j++) {
+		if (flag_stop)
+			break;
+
+		*flag = 0;
+
+		*afu_enable_reg_p = enable_in;
+		miso(); // force no gather
+
+		while (*flag == 0);
+	}
+	timebase[1] = read_timebase();
+
+	*count = j;
+	return (timebase[1] - timebase[0]);
+}
+
+// use '-m' option with value > 8 to use this function doing a 64/128B
+// MMIO write before the mmpp DMA write
+static inline uint64_t ping_OVER_8B(uint64_t global_mmio_start,
+				    volatile uint64_t *flag,
+				    uint64_t enable_in, uint64_t *count)
+{
+	uint64_t *afu_enable_reg_p, *afu_large_data0_p;
+	uint64_t timebase[2];
+	uint64_t i, j, num_dw, loop_count;
+
+	fprintf(stderr, "Use of ping data bigger than 8B requires special support in the ocxl driver for mmio write-combine. Disabled by default as it generates HMI on default setup\n");
+	*count = 1;
+	return 0;
+
+	afu_enable_reg_p  = (uint64_t *)(global_mmio_start +
+					 AFUEnable_AFP_REGISTER);
+	afu_large_data0_p = (uint64_t *)(global_mmio_start +
+					 Large_Data0_AFP_REGISTER);
+
+	num_dw = size_st / sizeof(uint64_t);
+
+	// Write enable register.
+	// The AFU is configured to use the register data for its DMA
+	// writes, so we need to make sure it's not 0 first.
+	for (i = 0; i < num_dw; i++)
+		*(afu_large_data0_p + i) = 0xCEECEECEECEE0000 + i;
+
+	*flag = 0;
+	*afu_enable_reg_p = (uint64_t) enable_in;
+	miso(); // force no gather
+
+	while (*flag == 0);
+
+	if (*count) {
+		loop_count = *count;
+	} else {
+		loop_count = ~0ull;
+		printf("Running test forever, interrupt with ctrl-c\n");
+	}
+
+	timebase[0] = read_timebase();
+	for (j = 0; j < loop_count; j++) {
+		if (flag_stop)
+			break;
+
+		*flag = 0;
+		// Write the large_data0 128 register
+		// num_dw = 8 if m64, num_dw = 16 if m128
+		for (i = 0; i < num_dw; i++) {
+			// Write large_data0  register
+			*(afu_large_data0_p + i) = 0xDAFADAFADAFA0000 + i;
+		}
+
+		miso(); // force no gather
+
+		while (*flag == 0);
+	}
+	timebase[1] = read_timebase();
+
+	*count = j;
+	return (timebase[1] - timebase[0]);
+}
+
+//Main function called after line commands arguments processed
+int ocapi_afp3_lat(void)
+{
+	int rc, j, k;
+	int size_enc_ld, size_enc_st;
+	int tags_ld = 0, tags_st = 7;
+	int npu_ld = 0, npu_st = 0;
+	int num_dw, use_large_data;
+	uint64_t total_latency, global_mmio_start, offsetmask;
+	uint64_t wed_in, misc_in, enable_in, extra_read_ea_in;
+	int flag_location;
+	volatile uint64_t *buffer;
+	size_t size;
+	ocxl_err err;
+	ocxl_afu_h afu_h;
+	ocxl_mmio_h mmio_h;
+
+	if (size_ping == 8)
+		use_large_data = 0;
+	else
+		use_large_data = 1;
+
+	size_ld = size_st;
+
+	switch (size_st) {
+	case 64:
+		size_enc_st = 1;
+		break;
+	case 128:
+		size_enc_st = 2;
+		break;
+	case 256:
+	case 512:
+		size_enc_st = 3;
+		break;
+	default:
+		printf("\nIllegal value entered for --size_st argument = %d!!!!\n", size_st);
+		return -1;
+	}
+
+	switch (size_ld) {
+	case 64:
+		size_enc_ld = 1;
+		break;
+	case 128:
+		size_enc_ld = 2;
+		break;
+	case 256:
+	case 512:
+		size_enc_ld = 3;
+		break;
+	default:
+		printf("\nIllegal value entered for --size_ld argument = %d!!!!\n", size_ld);
+		return -1;
+	}
+
+	if ((tags_ld != 0) || (tags_st == 0))
+		printf("WARNING: For MMIO ping-pong latency mode, it is recommended to enable stores (tags_st > 0), and disable loads (tags_ld = 0)\n");
+
+	printf("Parameters used: tags_ld=%d - size_ld=%d - tags_st=%d - size_st=%d\n",
+	       tags_ld, size_ld, tags_st, size_st);
+
+	// Open AFU device(s)
+	if (verbose)
+		printf("Calling ocxl_afu_open\n");
+	if (device)
+		err = ocxl_afu_open_from_dev(device, &afu_h);
+	else
+		err = ocxl_afu_open(AFU_NAME, &afu_h);
+	if (err != OCXL_OK) {
+		fprintf(stderr, "ocxl_afu_open() failed for %s, error %d\n",
+			device ? device : AFU_NAME, err);
+		return err;
+	}
+
+	// attach to afu - attach does not "start" the afu anymore
+	if (verbose)
+		printf("Calling ocxl_afu_attach\n");
+	err = ocxl_afu_attach(afu_h, OCXL_ATTACH_FLAGS_NONE);
+	if (err != OCXL_OK) {
+		fprintf(stderr, "ocxl_afu_attach: %d", err);
+		return err;
+	}
+
+	// map the mmio spaces
+	err = ocxl_mmio_map(afu_h, OCXL_GLOBAL_MMIO, &mmio_h);
+	if (err != OCXL_OK) {
+		fprintf(stderr, "ocxl_mmio_map: %d\n", err);
+		return err;
+	}
+
+	err = ocxl_mmio_get_info(mmio_h, (void **)&global_mmio_start, &size);
+	if (err != OCXL_OK) {
+		fprintf(stderr, "ocxl_mmio_get_info: %d\n", err);
+		return err;
+	}
+	printf("MMIO INFO: address 0x%016lx - size 0x%lx\n",
+	       global_mmio_start, size);
+
+	// Allocate a buffer for "to" memory buffer.
+	// Force alignment of address on cacheline boundary.
+	offsetmask = 0x7F << 12;  // Hardcode to 512K
+	rc = posix_memalign((void **) &buffer, BUF_512MB, BUF_512MB);
+	if (rc) {
+		perror("memalign main buffer");
+		return -1;
+	}
+	if (verbose)
+		printf("Allocated Buffer memory @ 0x%016llx\n",
+		       (long long)buffer);
+
+	// Turn off MMIO latency mode
+	err = ocxl_mmio_write64(mmio_h, AFUEnable_AFP_REGISTER,
+				OCXL_MMIO_LITTLE_ENDIAN, disableAfu);
+	if (err != OCXL_OK) {
+		fprintf(stderr,
+			"ocxl_mmio_write64(AFUEnable_AFP_REGISTER): %d\n", err);
+		return err;
+	}
+
+	// Initialize WED value
+	wed_in = (uint64_t) buffer +
+		(tags_ld << 9) + (size_enc_ld << 7) + (npu_ld << 6) +
+		(tags_st << 3) + (size_enc_st << 1) + (npu_st);
+	if (verbose)
+		printf("WED = %016lx\n", wed_in);
+	err = ocxl_mmio_write64(mmio_h, AFUWED_AFP_REGISTER,
+				OCXL_MMIO_LITTLE_ENDIAN, wed_in);
+	if (err != OCXL_OK) {
+		fprintf(stderr,
+			"ocxl_mmio_write64(AFUWED_AFP_REGISTER): %d\n", err);
+		return err;
+	}
+
+	if (verbose)
+		printf("BUFMASK = %016lx\n", offsetmask);
+	err = ocxl_mmio_write64(mmio_h, AFUBufmask_AFP_REGISTER,
+				OCXL_MMIO_LITTLE_ENDIAN, offsetmask);
+	if (err != OCXL_OK) {
+		fprintf(stderr,
+			"ocxl_mmio_write64(AFUBufmask_AFP_REGISTER): %d\n", err);
+		return err;
+	}
+
+	if (use_large_data) {
+		misc_in = 1 << 12; // 0b01: triggered by writing or
+				   // reading large data 0 register
+		if (verbose)
+			printf("MISC_REG = %016lx\n", misc_in);
+
+		err = ocxl_mmio_write64(mmio_h, AFUMisc_AFP_REGISTER,
+					OCXL_MMIO_LITTLE_ENDIAN, misc_in);
+		if (err != OCXL_OK) {
+			fprintf(stderr,
+				"ocxl_mmio_write64(AFUMisc_AFP_REGISTER): %d\n",
+				err);
+			return err;
+		}
+	}
+
+	if (verbose)
+		printf("CONTROL_REG (reset) = %016lx\n", resetCnt);
+	err = ocxl_mmio_write64(mmio_h, AFUControl_AFP_REGISTER,
+				OCXL_MMIO_LITTLE_ENDIAN, resetCnt);
+	if (err != OCXL_OK) {
+		fprintf(stderr, "ocxl_mmio_write64(AFUControl_AFP_REGISTER): %d\n", err);
+		return err;
+	}
+
+	if (extra_read) {
+		// Set Read address to base address + 1K.  This way,
+		// it does not overlap with MMIO Latency DMA Writes,
+		// and we do not need to set up more memory
+		extra_read_ea_in = (uint64_t) buffer + 1024;
+		if (verbose)
+			printf("EXTRA_READ_EA = %016lx\n", extra_read_ea_in);
+
+		err = ocxl_mmio_write64(mmio_h, AFUExtraReadEA_AFP_REGISTER,
+					OCXL_MMIO_LITTLE_ENDIAN,
+					extra_read_ea_in);
+		if (err != OCXL_OK) {
+			fprintf(stderr,
+				"ocxl_mmio_write64(AFUExtraReadEA_AFP_REGISTER): %d\n", err);
+			return err;
+		}
+
+		printf("Initializing extra_read memory .....\n");
+		for (j = 0; j < 64; j++)
+			buffer[(1024/8) + j] = 0xdafa0201dafa0100 + j;
+
+		if (verbose) {
+			printf("Done initializing extra read memory\n");
+			printf_buf(extra_read_ea_in, 512);
+		}
+	}
+
+	// Set ENABLE register
+	enable_in = BIT(63) | BIT(62); // AFU enable | MMIO ping pong latency test mode
+	if (size_st == 512)
+		enable_in |= BIT(61); // use 512B stores
+	if (use_large_data)
+		enable_in |= BIT(60); // use large ping pong data register for DMA write(s)
+	if (extra_read)
+		enable_in |= BIT(59); // extra read mode
+	if (size_ld == 512)
+		enable_in |= BIT(58); // use 512B loads
+	if (verbose) {
+		printf("ENABLE_REG = %016lx", enable_in);
+		if (use_large_data)
+			printf("\t> use large data regs value\n");
+		else
+			printf("\n");
+	}
+
+	num_dw = size_st / sizeof(uint64_t);
+	for (k = 0; k < num_dw; k++)
+		buffer[k] = 0;
+
+	if (verbose) {
+		printf("Buffer before test\n");
+		printf_buf((uint64_t) buffer, 512);
+	}
+
+	asm volatile("": : :"memory");
+	asm volatile("sync");
+
+	///////////////////////////////////////////////////////////////////////
+	// MMIO Ping-Pong Latency Test
+	///////////////////////////////////////////////////////////////////////
+
+	if (verbose)
+		printf("Calling ping_pong test\n");
+	printf("MMIO WR %dB (host to card) -> %sDMA WR %dB (card to host)\n",
+	       size_ping, (extra_read ? "DMA RD + " : ""), size_st);
+
+	// flag_location is the address where lower bytes of counter
+	// value will be set
+	flag_location = (size_st - 64) / sizeof(uint64_t);
+
+	if (size_ping == 8)
+		total_latency = ping_8B(global_mmio_start,
+					&buffer[flag_location], enable_in,
+					&iterations);
+	else
+		total_latency = ping_OVER_8B(global_mmio_start,
+					     &buffer[flag_location], enable_in,
+					     &iterations);
+
+	if (verbose) {
+		usleep(100000); // .1s
+		printf("\nBuffer after test\n");
+		printf_buf((uint64_t) buffer, 512);
+	}
+	printf("Completed %lu iterations. Total time measured using timebase: %10.2f ns\n",
+	       iterations, total_latency*1000./512);
+	printf("Average roundtrip per iteration: %10.2f ns\n",
+	       total_latency*1000./iterations/512);
+
+	// Turn off MMIO latency mode
+	err = ocxl_mmio_write64(mmio_h, AFUEnable_AFP_REGISTER,
+				OCXL_MMIO_LITTLE_ENDIAN, disableAfu);
+	if (err != OCXL_OK) {
+		fprintf(stderr,
+			"ocxl_mmio_write64(AFUEnable_AFP_REGISTER): %d\n", err);
+		return err;
+	}
+
+	if (verbose)
+		printf("Unmap afu\n");
+	ocxl_mmio_unmap(mmio_h);
+
+	if (verbose)
+		printf("Free afu\n");
+	ocxl_afu_close(afu_h);
+	return 0;
+}
+
+static void print_help(char *name)
+{
+	printf("\nUsage: %s [ options ]\n", name);
+	printf("\t-i 10000     --iterations\tDefault=%ld\n", iterations);
+	printf("\t-p 64 to 512 --pong      \tPong size from card to host (Bytes) Default=%d\n", size_st);
+	printf("\t-x           --extraread \tAdd an DMA extraread before the DMA Wr Default is no\n");
+	printf("\t-f           --forever   \tRun until CTRL+C, Default=no\n");
+	printf("\t-d           --device    \tDevice to open instead of first AFP AFU found\n");
+	printf("\t-v           --verbose   \tVerbose output\n");
+	printf("\t-h           --help      \tPrint this message\n");
+	printf("\n");
+}
+
+int main(int argc, char *argv[])
+{
+	int opt;
+	int option_index = 0;
+
+	static struct option long_options[] = {
+		{"iterations", required_argument, 0       , 'i'},
+		{"ping",       required_argument, 0       , 'm'},
+		{"pong",       required_argument, 0       , 'p'},
+		{"extraread",  no_argument      , 0       , 'x'},
+		{"forever",    no_argument      , 0       , 'f'},
+		{"verbose",    no_argument      , &verbose,  1 },
+		{"help",       no_argument      , 0       , 'h'},
+		{"device",     required_argument, 0       , 'd'},
+		{NULL, 0, 0, 0}
+	};
+
+	while ((opt = getopt_long(argc, argv, "vxhi:p:m:fd:", long_options,
+				  &option_index)) >= 0) {
+		switch (opt) {
+		case 'v':
+			verbose = 1;
+			break;
+		case 'i':
+			iterations = strtoul(optarg, NULL, 0);
+			break;
+		case 'm':
+			size_ping = strtoul(optarg, NULL, 0);
+			break;
+		case 'p':
+			size_st = strtoul(optarg, NULL, 0);
+			break;
+		case 'x':
+			extra_read = 1;
+			break;
+		case 'f':
+			iterations = 0;
+			break;
+		case 'h':
+			print_help(argv[0]);
+			return 0;
+		case 'd':
+			device = optarg;
+			break;
+		default:
+			print_help(argv[0]);
+			return -1;
+		}
+	}
+
+	// Registering signal handlers, useful for 'forever' mode
+	if (signal(SIGINT, stop_handler) == SIG_ERR)
+		printf("\ncan't catch SIGINT\n");
+	if (signal(SIGTERM, stop_handler) == SIG_ERR)
+		printf("\ncan't catch SIGTERM\n");
+
+	return ocapi_afp3_lat();
+}