New benchmark framework

with pluggable style benchmarks and an original SSH RTT calculator !

5 files changed, 399 insertions, 10 deletions

diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt
index eb843935..5b2b808d 100644
--- a/benchmarks/CMakeLists.txt
+++ b/benchmarks/CMakeLists.txt
@@ -1,16 +1,16 @@
 project(libssh-benchmarks C)
 
 set(benchmarks_SRCS
-  
+  bench_scp.c benchmarks.c latency.c
 )
 
 include_directories(
   ${LIBSSH_PUBLIC_INCLUDE_DIRS}
 )
 
-add_executable(bench_scp bench_scp.c ${benchmarks_SRCS})
+add_executable(benchmarks ${benchmarks_SRCS})
 
-target_link_libraries(bench_scp ${LIBSSH_SHARED_LIBRARY})
+target_link_libraries(benchmarks ${LIBSSH_SHARED_LIBRARY})
 
 include_directories(
   ${LIBSSH_PUBLIC_INCLUDE_DIRS}
diff --git a/benchmarks/bench_scp.c b/benchmarks/bench_scp.c
index 7406287d..3ef3bbfc 100644
--- a/benchmarks/bench_scp.c
+++ b/benchmarks/bench_scp.c
@@ -20,12 +20,6 @@
  * MA 02111-1307, USA.
  */
 
+#include "benchmarks.h"
 #include <libssh/libssh.h>
-#include <stdio.h>
 
-int main(int argc, char **argv){
-  (void) argc;
-  (void) argv;
-  printf("bench_scp\n");
-  return 0;
-}
diff --git a/benchmarks/benchmarks.c b/benchmarks/benchmarks.c
new file mode 100644
index 00000000..44ddb5d8
--- /dev/null
+++ b/benchmarks/benchmarks.c
@@ -0,0 +1,228 @@
+/*
+ * This file is part of the SSH Library
+ *
+ * Copyright (c) 2010 by Aris Adamantiadis
+ *
+ * The SSH Library is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at your
+ * option) any later version.
+ *
+ * The SSH Library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the SSH Library; see the file COPYING.  If not, write to
+ * the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
+ * MA 02111-1307, USA.
+ */
+
+#include "config.h"
+#include "benchmarks.h"
+#include <libssh/libssh.h>
+
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#ifdef HAVE_ARGP_H
+#include <argp.h>
+
+const char *argp_program_version = "libssh benchmarks 2010-12-28";
+const char *argp_program_bug_address = "Aris Adamantiadis <aris@0xbadc0de.be>";
+
+static char **cmdline;
+
+#define MAX_HOSTS_CONNECT 20
+
+/* Program documentation. */
+static char doc[] = "libssh benchmarks";
+enum libssh_benchmarks {
+    BENCHMARK_RAW_UPLOAD=1,
+    BENCHMARK_NUMBER
+};
+
+const char *libssh_benchmarks_names[]={
+    "null",
+    "benchmark_raw_upload"
+};
+
+struct argument_s {
+  const char *hosts[MAX_HOSTS_CONNECT];
+  char benchmarks[BENCHMARK_NUMBER -1];
+  int verbose;
+  int nhosts;
+  int ntests;
+};
+
+/* The options we understand. */
+static struct argp_option options[] = {
+  {
+    .name  = "verbose",
+    .key   = 'v',
+    .arg   = NULL,
+    .flags = 0,
+    .doc   = "Make libssh benchmark more verbose",
+    .group = 0
+  },
+  {
+    .name  = "raw-upload",
+    .key   = '1',
+    .arg   = NULL,
+    .flags = 0,
+    .doc   = "Upload raw data using channel",
+    .group = 0
+  },
+  {
+    .name  = "host",
+    .key   = 'h',
+    .arg   = "HOST",
+    .flags = 0,
+    .doc   = "Add a host to connect for benchmark (format user@hostname)",
+    .group = 0
+  },
+  {NULL, 0, NULL, 0, NULL, 0}
+};
+
+/* Parse a single option. */
+static error_t parse_opt (int key, char *arg, struct argp_state *state) {
+  /* Get the input argument from argp_parse, which we
+   * know is a pointer to our arguments structure.
+   */
+  struct argument_s *arguments = state->input;
+
+  /* arg is currently not used */
+  (void) arg;
+
+  switch (key) {
+    case '1':
+      arguments->benchmarks[key - '1' + 1] = 1;
+      arguments->ntests ++;
+      break;
+    case 'v':
+      arguments->verbose++;
+      break;
+    case 'h':
+      if(arguments->nhosts >= MAX_HOSTS_CONNECT){
+        fprintf(stderr, "Too much hosts\n");
+        return ARGP_ERR_UNKNOWN;
+      }
+      arguments->hosts[arguments->nhosts]=arg;
+      arguments->nhosts++;
+      break;
+    case ARGP_KEY_ARG:
+      /* End processing here. */
+      cmdline = &state->argv [state->next - 1];
+      state->next = state->argc;
+      break;
+    default:
+      return ARGP_ERR_UNKNOWN;
+  }
+
+  return 0;
+}
+
+/* Our argp parser. */
+static struct argp argp = {options, parse_opt, NULL, doc, NULL, NULL, NULL};
+
+#endif /* HAVE_ARGP_H */
+
+static void cmdline_parse(int argc, char **argv, struct argument_s *arguments) {
+  /*
+   * Parse our arguments; every option seen by parse_opt will
+   * be reflected in arguments.
+   */
+#ifdef HAVE_ARGP_H
+  argp_parse(&argp, argc, argv, 0, 0, arguments);
+#endif /* HAVE_ARGP_H */
+}
+
+static void arguments_init(struct argument_s *arguments){
+  memset(arguments,0,sizeof(*arguments));
+}
+
+static ssh_session connect_host(const char *host, int verbose){
+  ssh_session session=ssh_new();
+  if(session==NULL)
+    goto error;
+  if(ssh_options_set(session,SSH_OPTIONS_HOST, host)<0)
+    goto error;
+  ssh_options_set(session, SSH_OPTIONS_LOG_VERBOSITY, &verbose);
+  if(ssh_connect(session)==SSH_ERROR)
+    goto error;
+  if(ssh_userauth_autopubkey(session,NULL) != SSH_AUTH_SUCCESS)
+    goto error;
+  return session;
+error:
+  fprintf(stderr,"Error connecting to \"%s\": %s\n",host,ssh_get_error(session));
+  ssh_free(session);
+  return NULL;
+}
+
+static void do_benchmarks(ssh_session session, struct argument_s *arguments,
+    const char *hostname){
+  float ping_rtt=0.0;
+  float ssh_rtt=0.0;
+  int err;
+
+  if(arguments->verbose>0)
+    fprintf(stdout,"Testing ICMP RTT\n");
+  err=benchmarks_ping_latency(hostname, &ping_rtt);
+  if(err == 0){
+    fprintf(stdout,"ping RTT : %f ms\n",ping_rtt);
+  }
+  err=benchmarks_ssh_latency(session, &ssh_rtt);
+  if(err==0){
+    fprintf(stdout, "SSH RTT : %f ms\n",ssh_rtt);
+  }
+}
+
+int main(int argc, char **argv){
+  struct argument_s arguments;
+  ssh_session session;
+  int i;
+
+  arguments_init(&arguments);
+  cmdline_parse(argc, argv, &arguments);
+  if (arguments.nhosts==0){
+    fprintf(stderr,"At least one host must be specified");
+    return EXIT_FAILURE;
+  }
+  if (arguments.ntests==0){
+    for(i=1; i < BENCHMARK_NUMBER ; ++i){
+      arguments.benchmarks[i-1]=1;
+    }
+    arguments.ntests=BENCHMARK_NUMBER-1;
+  }
+  if (arguments.verbose > 0){
+    fprintf(stdout, "Will try hosts ");
+    for(i=0;i<arguments.nhosts;++i){
+      fprintf(stdout,"\"%s\" ", arguments.hosts[i]);
+    }
+    fprintf(stdout,"with benchmarks ");
+    for(i=0;i<BENCHMARK_NUMBER-1;++i){
+      if(arguments.benchmarks[i])
+        fprintf(stdout,"\"%s\" ",libssh_benchmarks_names[i+1]);
+    }
+    fprintf(stdout,"\n");
+  }
+
+  for(i=0; i<arguments.nhosts;++i){
+    if(arguments.verbose > 0)
+      fprintf(stdout,"Connecting to \"%s\"...\n",arguments.hosts[i]);
+    session=connect_host(arguments.hosts[i], arguments.verbose);
+    if(session != NULL && arguments.verbose > 0)
+      fprintf(stdout,"Success\n");
+    if(session == NULL){
+      fprintf(stderr,"Errors occured, stopping\n");
+      return EXIT_FAILURE;
+    }
+    do_benchmarks(session, &arguments, arguments.hosts[i]);
+    ssh_disconnect(session);
+    ssh_free(session);
+  }
+  return EXIT_SUCCESS;
+}
+
diff --git a/benchmarks/benchmarks.h b/benchmarks/benchmarks.h
new file mode 100644
index 00000000..fa32c9c9
--- /dev/null
+++ b/benchmarks/benchmarks.h
@@ -0,0 +1,31 @@
+/*
+ * This file is part of the SSH Library
+ *
+ * Copyright (c) 2010 by Aris Adamantiadis
+ *
+ * The SSH Library is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at your
+ * option) any later version.
+ *
+ * The SSH Library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the SSH Library; see the file COPYING.  If not, write to
+ * the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
+ * MA 02111-1307, USA.
+ */
+
+#ifndef BENCHMARKS_H_
+#define BENCHMARKS_H_
+
+#include <libssh/libssh.h>
+
+/* latency.c */
+int benchmarks_ping_latency (const char *host, float *average);
+int benchmarks_ssh_latency (ssh_session session, float *average);
+
+#endif /* BENCHMARKS_H_ */
diff --git a/benchmarks/latency.c b/benchmarks/latency.c
new file mode 100644
index 00000000..17379453
--- /dev/null
+++ b/benchmarks/latency.c
@@ -0,0 +1,136 @@
+/*
+ * This file is part of the SSH Library
+ *
+ * Copyright (c) 2010 by Aris Adamantiadis
+ *
+ * The SSH Library is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation; either version 2.1 of the License, or (at your
+ * option) any later version.
+ *
+ * The SSH Library is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with the SSH Library; see the file COPYING.  If not, write to
+ * the Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
+ * MA 02111-1307, USA.
+ */
+
+#include "benchmarks.h"
+#include <libssh/libssh.h>
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <sys/time.h>
+
+#define PING_PROGRAM "/bin/ping"
+
+/** @internal
+ * @brief Calculates the RTT of the host with ICMP ping, and returns the
+ * average of the calculated RTT.
+ * @param[in] host hostname to ping.
+ * @param[out] average average RTT in ms.
+ * @returns 0 on success, -1 if there is an error.
+ * @warning relies on an external ping program which may not exist on
+ * certain OS.
+ */
+int benchmarks_ping_latency (const char *host, float *average){
+  const char *ptr;
+  char cmd[256];
+  char line[1024];
+  FILE *fd;
+  int found=0;
+
+  /* strip out the hostname */
+  ptr=strchr(host,'@');
+  if(ptr)
+    ptr++;
+  else
+    ptr=host;
+
+  snprintf(cmd,sizeof(cmd),"%s -n -q -c3 %s",PING_PROGRAM, ptr);
+  fd=popen(cmd,"r");
+  if(fd==NULL){
+    fprintf(stderr,"Error executing command : %s\n", strerror(errno));
+    return -1;
+  }
+
+  while(!found && fgets(line,sizeof(line),fd)!=NULL){
+    if(strstr(line,"rtt")){
+      ptr=strchr(line,'=');
+      if(ptr==NULL)
+        goto parseerror;
+      ptr=strchr(ptr,'/');
+      if(ptr==NULL)
+        goto parseerror;
+      *average=strtof(ptr+1,NULL);
+      found=1;
+      break;
+    }
+  }
+  if(!found)
+    goto parseerror;
+  fclose(fd);
+  return 0;
+
+parseerror:
+  fprintf(stderr,"Parse error : couldn't locate average in %s",line);
+  fclose(fd);
+  return -1;
+}
+
+struct timestamp_struct {
+  struct timeval timestamp;
+};
+
+static void timestamp_init(struct timestamp_struct *ts){
+  gettimeofday(&ts->timestamp,NULL);
+}
+
+static float elapsed_time(struct timestamp_struct *ts){
+  struct timeval now;
+  time_t secdiff;
+  long usecdiff; /* may be negative */
+
+  gettimeofday(&now,NULL);
+  secdiff=now.tv_sec - ts->timestamp.tv_sec;
+  usecdiff=now.tv_usec - ts->timestamp.tv_usec;
+  //printf("%d sec diff, %d usec diff\n",secdiff, usecdiff);
+  return (float) (secdiff*1000) + ((float)usecdiff)/1000;
+}
+
+int benchmarks_ssh_latency(ssh_session session, float *average){
+  float times[3];
+  struct timestamp_struct ts;
+  int i;
+  ssh_channel channel;
+  channel=ssh_channel_new(session);
+  if(channel==NULL)
+    goto error;
+  if(ssh_channel_open_session(channel)==SSH_ERROR)
+    goto error;
+
+  for(i=0;i<3;++i){
+    timestamp_init(&ts);
+    if(ssh_channel_request_env(channel,"TEST","test")==SSH_ERROR &&
+        ssh_get_error_code(session)==SSH_FATAL)
+      goto error;
+    times[i]=elapsed_time(&ts);
+  }
+  ssh_channel_close(channel);
+  ssh_channel_free(channel);
+  channel=NULL;
+  printf("Times : %f ms ; %f ms ; %f ms\n", times[0], times[1], times[2]);
+  *average=(times[0]+times[1]+times[2])/3;
+  return 0;
+error:
+  fprintf(stderr,"Error calculating SSH latency : %s\n",ssh_get_error(session));
+  if(channel)
+    ssh_channel_free(channel);
+  return -1;
+}