bug fix for splice, add normal udp + onecopy

seems cannot combine bypassl3 with one copy.. need more digging
This commit is contained in:
HappyZ 2017-02-26 15:04:03 -08:00
parent 0d9604e37e
commit 51a5036f0e
15 changed files with 688 additions and 1 deletions

View File

@ -48,7 +48,7 @@ int main(int argc, char *argv[])
uint quota = 1000000000; // default bytes per slot, default 1GB/slot uint quota = 1000000000; // default bytes per slot, default 1GB/slot
uint sentInSlot = 0, slot = 1; uint sentInSlot = 0, slot = 1;
uint total_bytes_sent = 0; uint total_bytes_sent = 0;
uint bytes, bytes_sent, bytes_in_pipe; int bytes, bytes_sent, bytes_in_pipe;
// for timing // for timing
double elapsedTime; double elapsedTime;
struct timeval t_start, t_end, t_now; struct timeval t_start, t_end, t_now;

View File

@ -0,0 +1,18 @@
#CC = /media/Lucifer/android/lib/android-18-toolchain/bin/arm-linux-androideabi-gcc
CC1 = arm-linux-androideabi-gcc
CC2 = gcc
CFLAGS = -pie -fPIE
TARGET1 = client_send_normaludp_sendfile
TARGET2 = client_send_normaludp_sendfile_pc
all: $(TARGET1) $(TARGET2)
$(TARGET1): main.c
$(CC1) main.c -Wall $(CFLAGS) -o $@
$(TARGET2): main.c
$(CC2) main.c -Wall $(CFLAGS) -o $@
clean:
-rm -f $(TARGET1)
-rm -f $(TARGET2)

View File

@ -0,0 +1,250 @@
/*
* Initial commit by Yibo @ Jul. 28, 2015
* Last update by Yanzi @ Sept. 26, 2016
*/
#include <stdio.h>
#include <math.h>
#include <string.h>
#include <arpa/inet.h>
#include <linux/if_packet.h>
#include <stdlib.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <net/if.h>
#include <netinet/ether.h>
#include <sys/sendfile.h>
#include <errno.h>
#include <sys/time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <fcntl.h>
#define BUF_SIZ 65536
#if !defined SPLICE_F_MORE
# define SPLICE_F_MOVE 1 /* Move pages instead of copying. */
# define SPLICE_F_NONBLOCK 2 /* Don't block on the pipe splicing
(but we may still block on the fd
we splice from/to). */
# define SPLICE_F_MORE 4 /* Expect more data. */
# define SPLICE_F_GIFT 8 /* Pages passed in are a gift. */
#endif
char isNumber(char number[])
{
int i = 0;
//checking for negative numbers
if (number[0] == '-')
i = 1;
for (; number[i] != 0; i++)
{
//if (number[i] > '9' || number[i] < '0')
if (!isdigit(number[i]))
return 0;
}
return 1;
}
int main(int argc, char *argv[])
{
// for bandwidth control
uint slotLength = 10000; // in microseconds, for bandwidth control
uint quota = 1000000000; // default bytes per slot, default 1GB/slot
uint sentInSlot = 0, slot = 1;
uint total_bytes_sent = 0;
int bytes, bytes_sent, bytes_in_pipe;
// for timing
double elapsedTime;
struct timeval t_start, t_end, t_now;
// for socket
int fd; // file descriptor of file to send
int sockfd; // socket
// char ifName[IFNAMSIZ];
// char sendbuf[BUF_SIZ];
struct sockaddr_in servaddr;
// struct ether_header *eh = (struct ether_header *) sendbuf;
// struct iphdr *iph = (struct iphdr *) (sendbuf + sizeof(struct ether_header));
// struct sockaddr_ll socket_address;
// for misc
int ret;
int sendsize = 1472; // 1500 MTU - 20 IPv4 - 8 UDP
int bytes2send = 0;
struct stat st;
off_t offset = 0;
// create two pipes
int filedes[2];
ret = pipe(filedes);
if (argc < 4)
{
printf("Usage: %s <bytes2send/file2send> <ip> <port> <[optional] bandwidth (bps)> <[optional] sendsize (bytes)>\n", argv[0]);
exit(0);
}
// set bandwidth
if (argc > 4)
quota = atoi(argv[4]) / 8 / (1000000 / slotLength);
// set sendsize (if larger than 1472 will do packetization (fragmentation) (is this true??))
if (argc > 5)
sendsize = atoi(argv[5]);
// adjust slotLength to address packet size issue in the end
if ((quota % sendsize) > 0)
{
// printf("quota:%d,sendsize:%d,slotLength:%d\n", quota, sendsize, slotLength);
slotLength = (uint)((double)(quota / sendsize + 1) * sendsize / quota * slotLength);
quota = (quota / sendsize + 1) * sendsize;
// printf("quota:%d,sendsize:%d,slotLength:%d\n", quota, sendsize, slotLength);
}
// get file size (bytes2send)
if (isNumber(argv[1]))
{
// set bytes to send
bytes2send = atoi(argv[1]);
// open file descriptor
fd = open("/data/local/tmp/bigfile", O_RDONLY);
if (fd == -1)
{
fprintf(stderr, "! Unable to open /data/local/tmp/bigfile.\n");
exit(1);
}
}
else
{
// open file descriptor
fd = open(argv[1], O_RDONLY);
if (fd == -1)
{
fprintf(stderr, "! Unable to open file %s.\n", argv[1]);
exit(1);
}
fstat(fd, &st);
bytes2send = st.st_size;
printf("bytes2send:%d\n", bytes2send);
}
// bind socket
sockfd = socket(AF_INET, SOCK_DGRAM, 0);
bzero(&servaddr, sizeof(servaddr));
servaddr.sin_family = AF_INET;
servaddr.sin_addr.s_addr = inet_addr(argv[2]);
servaddr.sin_port = htons(atoi(argv[3]));
if(connect(sockfd, (struct sockaddr *) &servaddr, sizeof(servaddr)) < 0) {
perror("connect failed\n");
exit(-1);
}
// start timing
gettimeofday(&t_start, NULL);
// start to send
while (total_bytes_sent < bytes2send)
{
if ((bytes2send - total_bytes_sent) < quota)
{
quota = bytes2send - total_bytes_sent;
}
// initialize ret
// ret = 1;
// send in slots
while (sentInSlot < quota)
{
// printf(
// "before: total_bytes_sent %d, sentInSlot %d, quota - sentInSlot %d\n",
// total_bytes_sent, sentInSlot, quota - sentInSlot);
ret = sendfile(sockfd, fd, &offset,
((quota - sentInSlot < sendsize) ? (quota - sentInSlot) : sendsize));
// // Splice the data from in_fd into the pipe
// if ((bytes_sent = splice(fd, NULL, filedes[1], NULL,
// (quota - sentInSlot < sendsize) ? (quota - sentInSlot) : sendsize,
// SPLICE_F_MORE | SPLICE_F_MOVE)) <= 0) {
// if (errno == EINTR || errno == EAGAIN) {
// // Interrupted system call/try again
// // Just skip to the top of the loop and try again
// usleep(100);
// continue;
// }
// fprintf(stderr, "! splice error, errno: %d.\n", errno);
// exit(1);
// }
// // Splice the data from the pipe into out_fd
// bytes_in_pipe = bytes_sent;
// printf("bytes_in_pipe %d, err:%d\n", (int)bytes_sent, errno);
// while (bytes_in_pipe > 0) {
// if ((bytes = splice(filedes[0], NULL, sockfd, NULL, bytes_in_pipe,
// SPLICE_F_MORE | SPLICE_F_MOVE)) <= 0) {
// if (errno == EINTR || errno == EAGAIN || errno == EMSGSIZE) {
// // Interrupted system call/try again
// // Just skip to the top of the loop and try again
// fprintf(stderr, "! sleep 100, err:%d.\n", errno);
// usleep(1000);
// continue;
// }
// fprintf(stderr, "! splice error, errno: %d.\n", errno);
// usleep(1000);
// continue;
// }
// bytes_in_pipe -= bytes;
// // printf("bytes_in_pipe %d, value %d, err:%d\n", (int)bytes_in_pipe, (int)bytes, errno);
// }
// total_bytes_sent += bytes_sent;
// sentInSlot += bytes_sent;
if (ret <= 0)
{
fprintf(stderr, "! Fail to send: ret:%d, err:%d; wait for 100us..\n", ret, errno);
usleep(100);
// offset -= ((quota - sentInSlot < sendsize) ? (quota - sentInSlot) : sendsize);
continue;
}
sentInSlot += ret;
total_bytes_sent += ret;
// printf(
// "total_bytes_sent %d, sentInSlot %d, quota - sentInSlot %d\n",
// total_bytes_sent, sentInSlot, quota - sentInSlot);
}
// control bandwidth
gettimeofday(&t_now, NULL);
elapsedTime = (t_now.tv_sec - t_start.tv_sec) * 1000000.0 + (t_now.tv_usec - t_start.tv_usec);
if (elapsedTime < slotLength * slot)
{
// printf(
// "sent %d, quota %d, bytes2send %d, usleep %lfus\n",
// total_bytes_sent, quota, bytes2send, slotLength * slot - elapsedTime);
usleep((int)(slotLength * slot - elapsedTime));
}
sentInSlot = 0;
++slot;
}
ret = sendto(sockfd, "=?!\n", 4, 0, (struct sockaddr *)&servaddr, sizeof(servaddr));
if (ret <= 0)
{
fprintf(stderr, "! Unable to end data transfer. errno:%d.\n", errno);
close(sockfd);
close(fd);
exit(1);
}
// end timing
gettimeofday(&t_end, NULL);
elapsedTime = (t_end.tv_sec - t_start.tv_sec) + (t_end.tv_usec - t_start.tv_usec) / 1000000.0;
printf(
"sent(bytes):%d\nduration(s):%lf\nthroughput(bps):%lf\n",
total_bytes_sent, elapsedTime, total_bytes_sent * 8 / elapsedTime);
close(sockfd);
close(fd);
return 0;
}

View File

@ -0,0 +1,18 @@
#CC = /media/Lucifer/android/lib/android-18-toolchain/bin/arm-linux-androideabi-gcc
CC1 = arm-linux-androideabi-gcc
CC2 = gcc
CFLAGS = -pie -fPIE
TARGET1 = client_send_bypassl3_sendfile
TARGET2 = client_send_bypassl3_sendfile_pc
all: $(TARGET1) $(TARGET2)
$(TARGET1): main.c
$(CC1) main.c -Wall $(CFLAGS) -o $@
$(TARGET2): main.c
$(CC2) main.c -Wall $(CFLAGS) -o $@
clean:
-rm -f $(TARGET1)
-rm -f $(TARGET2)

View File

@ -0,0 +1,288 @@
/*
* Initial commit by Yibo @ Jul. 28, 2015
* Last update by Yanzi @ Sept. 26, 2016
*/
#include <stdio.h>
#include <string.h>
#include <arpa/inet.h>
#include <linux/if_packet.h>
#include <stdlib.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <net/if.h>
#include <netinet/ether.h>
#include <sys/sendfile.h>
#include <errno.h>
#include <sys/time.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <fcntl.h>
#define ETH_P_IP 0x0800 /* Internet Protocol packet */
#define ETH_ALEN 6 /* from <net/ethernet.h> */
#define ETH_P_ALL 0x0003
#define MY_DEST_MAC0 0x18
#define MY_DEST_MAC1 0x03
#define MY_DEST_MAC2 0x73
#define MY_DEST_MAC3 0xc8
#define MY_DEST_MAC4 0x86
#define MY_DEST_MAC5 0x52
#define DEFAULT_IF "wlan0"
#define BUF_SIZ 4096
char isNumber(char number[])
{
int i = 0;
//checking for negative numbers
if (number[0] == '-')
i = 1;
for (; number[i] != 0; i++)
{
//if (number[i] > '9' || number[i] < '0')
if (!isdigit(number[i]))
return 0;
}
return 1;
}
int main(int argc, char *argv[])
{
// defaults
uint slotLength = 10000; // in microseconds, for bandwidth control
uint quota = 1000000000; // default bytes per slot, default 1GB/slot
uint sentInSlot = 0, slot = 1;
uint total_bytes_sent = 0;
off_t offset = 0;
// for timing
double elapsedTime;
struct timeval t_start, t_end, t_now;
// for socket
int i, j, fd; // file descriptor of file to send
int sockfd; // socket
int tx_len = 0;
char sendbuf[BUF_SIZ];
struct ifreq if_idx;
struct ifreq if_mac;
struct sockaddr_in servaddr;
struct ether_header *eh = (struct ether_header *) sendbuf;
// struct iphdr *iph = (struct iphdr *) (sendbuf + sizeof(struct ether_header));
struct sockaddr_ll socket_address;
char ifName[IFNAMSIZ];
// for misc
int ret;
int sendsize = 1500; // 1500 MTU (raw socket)
int bytes2send = 0;
struct stat st;
unsigned char my_dest_mac[6];
if (argc < 3)
{
printf("Usage: %s <bytes2send/file2send> <dest MAC address> <[optional] bandwidth (bps)> <[optional] sendsize (bytes)> <[optional] interface>\n", argv[0]);
exit(0);
}
// set bandwidth
if (argc > 3)
quota = atoi(argv[3]) / 8 / (1000000 / slotLength);
// set sendsize (if larger than 1460 will do packetization (fragmentation))
if (argc > 4)
sendsize = atoi(argv[4]);
// set interface
if (argc > 5) {
strcpy(ifName, argv[5]);
} else {
strcpy(ifName, DEFAULT_IF);
}
// adjust slotLength to address packet size issue in the end
if ((quota % sendsize) > 0)
{
// printf("quota:%d,sendsize:%d,slotLength:%d\n", quota, sendsize, slotLength);
slotLength = (uint)((double)(quota / sendsize + 1) * sendsize / quota * slotLength);
quota = (quota / sendsize + 1) * sendsize;
// printf("quota:%d,sendsize:%d,slotLength:%d\n", quota, sendsize, slotLength);
}
// get file size (bytes2send)
if (isNumber(argv[1]))
{
// set bytes to send
bytes2send = atoi(argv[1]);
// open file descriptor
fd = open("/data/local/tmp/bigfile", O_RDONLY);
if (fd == -1)
{
fprintf(stderr, "! Unable to open /data/local/tmp/bigfile.\n");
exit(1);
}
}
else
{
// open file descriptor
fd = open(argv[1], O_RDONLY);
if (fd == -1)
{
fprintf(stderr, "! Unable to open file %s.\n", argv[1]);
exit(1);
}
fstat(fd, &st);
bytes2send = st.st_size;
printf("bytes2send:%d\n", bytes2send);
}
if ((sockfd = socket(AF_PACKET, SOCK_RAW, IPPROTO_RAW)) == -1)
{
fprintf(stderr, "! raw socket error.\n");
exit(1);
}
// Get the index of the interface to send on
memset(&if_idx, 0, sizeof(struct ifreq));
strncpy(if_idx.ifr_name, ifName, IFNAMSIZ - 1);
if (ioctl(sockfd, SIOCGIFINDEX, &if_idx) < 0)
{
fprintf(stderr, "! SIOCGIFINDEX error. Check permission.\n");
exit(1);
}
// Get the MAC address of the interface to send on
memset(&if_mac, 0, sizeof(struct ifreq));
strncpy(if_mac.ifr_name, ifName, IFNAMSIZ - 1);
if (ioctl(sockfd, SIOCGIFHWADDR, &if_mac) < 0)
{
fprintf(stderr, "! SIOCGIFHWADDR error. Check permission.\n");
exit(1);
}
// Set mem
memset(sendbuf, 0, BUF_SIZ);
// parse input MAC address
sscanf(
argv[2], "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx",
&my_dest_mac[0], &my_dest_mac[1], &my_dest_mac[2],
&my_dest_mac[3], &my_dest_mac[4], &my_dest_mac[5]);
printf("destMAC:%02x:%02x:%02x:%02x:%02x:%02x\n",
my_dest_mac[0], my_dest_mac[1], my_dest_mac[2],
my_dest_mac[3], my_dest_mac[4], my_dest_mac[5]);
// Construct the Ethernet header
eh->ether_shost[0] = ((uint8_t *)&if_mac.ifr_hwaddr.sa_data)[0];
eh->ether_shost[1] = ((uint8_t *)&if_mac.ifr_hwaddr.sa_data)[1];
eh->ether_shost[2] = ((uint8_t *)&if_mac.ifr_hwaddr.sa_data)[2];
eh->ether_shost[3] = ((uint8_t *)&if_mac.ifr_hwaddr.sa_data)[3];
eh->ether_shost[4] = ((uint8_t *)&if_mac.ifr_hwaddr.sa_data)[4];
eh->ether_shost[5] = ((uint8_t *)&if_mac.ifr_hwaddr.sa_data)[5];
eh->ether_dhost[0] = my_dest_mac[0];
eh->ether_dhost[1] = my_dest_mac[1];
eh->ether_dhost[2] = my_dest_mac[2];
eh->ether_dhost[3] = my_dest_mac[3];
eh->ether_dhost[4] = my_dest_mac[4];
eh->ether_dhost[5] = my_dest_mac[5];
eh->ether_type = htons(ETH_P_IP);
tx_len += sizeof(struct ether_header);
// Index of the network device
socket_address.sll_ifindex = if_idx.ifr_ifindex;
// Address length
socket_address.sll_halen = ETH_ALEN;
// Destination MAC
socket_address.sll_addr[0] = MY_DEST_MAC0;
socket_address.sll_addr[1] = MY_DEST_MAC1;
socket_address.sll_addr[2] = MY_DEST_MAC2;
socket_address.sll_addr[3] = MY_DEST_MAC3;
socket_address.sll_addr[4] = MY_DEST_MAC4;
socket_address.sll_addr[5] = MY_DEST_MAC5;
if(bind(sockfd, (struct sockaddr *) &socket_address, sizeof(struct sockaddr_ll)) < 0) {
perror("bind\n");
exit(-1);
}
ret = sendto(
sockfd, sendbuf, sendsize,
MSG_DONTWAIT, (struct sockaddr*)&socket_address, sizeof(struct sockaddr_ll));
// fprintf(stderr, "! Fail to send: ret:%d, err:%d; wait for 100us..\n", ret, errno);
// start timing
gettimeofday(&t_start, NULL);
// read(fd, sendbuf + tx_len, sendsize - tx_len);
// start to send
while (total_bytes_sent < bytes2send)
{
if ((bytes2send - total_bytes_sent) < quota)
{
quota = bytes2send - total_bytes_sent;
}
// send in slots
while (sentInSlot < quota)
{
// printf(
// "before: total_bytes_sent %d, sentInSlot %d, quota - sentInSlot %d\n",
// total_bytes_sent, sentInSlot, quota - sentInSlot);
ret = sendfile(sockfd, fd, &offset,
((quota - sentInSlot < (sendsize - tx_len)) ? (quota - sentInSlot) : (sendsize - tx_len)));
// read(
// fd, sendbuf + tx_len,
// (
// ((quota - sentInSlot) < (sendsize - tx_len)) ?
// (quota - sentInSlot) : (sendsize - tx_len))
// );
// ret = sendto(
// sockfd, sendbuf,
// (quota - sentInSlot < sendsize) ? (quota - sentInSlot) : sendsize,
// MSG_DONTWAIT, (struct sockaddr*)&socket_address, sizeof(struct sockaddr_ll));
if (ret <= 0)
{
fprintf(stderr, "! Fail to send: ret:%d, err:%d; wait for 100us..\n", ret, errno);
usleep(100);
continue;
}
total_bytes_sent += ret;
sentInSlot += ret;
// printf(
// "after: total_bytes_sent %d, sentInSlot %d, quota - sentInSlot %d\n",
// total_bytes_sent, sentInSlot, quota - sentInSlot);
}
// control bandwidth
gettimeofday(&t_now, NULL);
elapsedTime = (t_now.tv_sec - t_start.tv_sec) * 1000000.0 + (t_now.tv_usec - t_start.tv_usec);
if (elapsedTime < (slotLength * slot))
{
// printf(
// "sent %d, quota %d, bytes2send %d, usleep %lfus\n",
// total_bytes_sent, quota, bytes2send, slotLength * slot - elapsedTime);
usleep((int)(slotLength * slot - elapsedTime));
}
sentInSlot = 0;
++slot;
}
// end timing
gettimeofday(&t_end, NULL);
elapsedTime = (t_end.tv_sec - t_start.tv_sec) + (t_end.tv_usec - t_start.tv_usec) / 1000000.0;
printf(
"sent(bytes):%d\nduration(s):%lf\nthroughput(bps):%lf\n",
total_bytes_sent, elapsedTime, total_bytes_sent * 8 / elapsedTime);
close(sockfd);
close(fd);
return 0;
}

Binary file not shown.

View File

@ -0,0 +1,113 @@
/*
* Copyright (C) 2009 Max Kellermann <max@duempel.org>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the
* distribution.
*/
/*
* This tiny program prints a matrix: which file descriptor
* combinations are supported by splice()?
*/
#define _GNU_SOURCE
#include <fcntl.h>
#include <errno.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/socket.h>
static struct {
const char *const name;
int in, out;
} fds[] = {
{ .name = "pipe", },
{ .name = "reg", },
{ .name = "chr", },
{ .name = "unix", },
{ .name = "tcp", },
{ .name = "udp", },
};
enum {
NUM_FDS = sizeof(fds) / sizeof(fds[0]),
};
int main(int argc, char **argv)
{
int f[2], ret;
unsigned x, y;
char template1[] = "/tmp/test_splice.XXXXXX";
char template2[] = "/tmp/test_splice.XXXXXX";
(void)argc;
(void)argv;
/* open two file descriptors of each kind */
fds[0].in = pipe(f) >= 0 ? f[0] : -1;
fds[0].out = pipe(f) >= 0 ? f[1] : -1;
fds[1].in = mkstemp(template1);
fds[1].out = mkstemp(template2);
fds[2].in = open("/dev/zero", O_RDONLY);
fds[2].out = open("/dev/null", O_WRONLY);
fds[3].in = socketpair(AF_UNIX, SOCK_STREAM, 0, f) >= 0 ? f[0] : -1;
fds[3].out = socketpair(AF_UNIX, SOCK_STREAM, 0, f) >= 0 ? f[0] : -1;
fds[4].in = socket(AF_INET, SOCK_STREAM, 0);
fds[4].out = socket(AF_INET, SOCK_STREAM, 0);
fds[5].in = socket(AF_INET, SOCK_DGRAM, 0);
fds[5].out = socket(AF_INET, SOCK_DGRAM, 0);
/* print table header */
printf("in\\out");
for (x = 0; x < NUM_FDS; ++x)
printf("\t%s", fds[x].name);
putchar('\n');
for (y = 0; y < NUM_FDS; ++y) {
fputs(fds[y].name, stdout);
for (x = 0; x < NUM_FDS; ++x) {
putchar('\t');
if (fds[x].out < 0 || fds[y].in < 0) {
fputs("n/a", stdout);
continue;
}
ret = splice(fds[y].in, NULL, fds[x].out, NULL, 1,
SPLICE_F_NONBLOCK);
if (ret >= 0 || errno == EAGAIN || errno == EWOULDBLOCK
|| errno == ENOTCONN)
/* EAGAIN or EWOULDBLOCK means that the kernel has
accepted this combination, but can't move pages
right now */
fputs("yes", stdout);
else if (errno == EINVAL)
/* the kernel doesn't support this combination */
fputs("no", stdout);
else if (errno == ENOSYS)
/* splice() isn't supported at all */
fputs("ENOSYS", stdout);
else
/* an unexpected error code */
fputs("err", stdout);
}
putchar('\n');
}
unlink(template1);
unlink(template2);
}