Add text format support, C++, and visualizations to perf harness. (#185)

* Add text format to perf harness and add visualizations.
* Add C++ harness and reörganize the visualizations a bit.
* Increase the number of measurements from 5 to 10 to get better noise
  filtering.
* Open instruments and viz in background to be less disruptive.
* Add "all" option as a field type to make multiple runs easier.

Performance/Harness.cc

@@ -0,0 +1,74 @@
+// Performance/Harness.cc - C++ performance harness definition
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+// -----------------------------------------------------------------------------
+///
+/// Defines the class that runs the performance tests.
+///
+// -----------------------------------------------------------------------------
+
+#include <chrono>
+#include <cstdio>
+#include <cmath>
+#include <iostream>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include "Harness.h"
+
+using std::chrono::duration_cast;
+using std::chrono::steady_clock;
+using std::endl;
+using std::function;
+using std::ostream;
+using std::result_of;
+using std::sqrt;
+using std::string;
+using std::vector;
+
+Harness::Harness(std::ostream* results_stream) :
+    results_stream(results_stream),
+    measurement_count(10),
+    run_count(100),
+    repeated_count(100) {}
+
+void Harness::write_to_log(const string& name,
+                           const vector<milliseconds_d>& timings) const {
+  if (results_stream == nullptr) {
+    return;
+  }
+
+  (*results_stream) << "\"" << name << "\": [";
+  for (const auto& duration : timings) {
+    auto millis = duration_cast<milliseconds_d>(duration);
+    (*results_stream) << millis.count() << ", ";
+  }
+  (*results_stream) << "]," << endl;
+}
+
+Harness::Statistics Harness::compute_statistics(
+    const vector<steady_clock::duration>& timings) const {
+  milliseconds_d::rep sum = 0;
+  milliseconds_d::rep sqsum = 0;
+
+  for (const auto& duration : timings) {
+    auto millis = duration_cast<milliseconds_d>(duration);
+    auto count = millis.count();
+    sum += count;
+    sqsum += count * count;
+  }
+
+  auto n = timings.size();
+  Statistics stats;
+  stats.mean = sum / n;
+  stats.stddev = sqrt(sqsum / n - stats.mean * stats.mean);
+  return stats;
+}

Performance/Harness.h

@@ -0,0 +1,161 @@
+// Performance/Harness.h - C++ performance harness declaration
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+// -----------------------------------------------------------------------------
+///
+/// Declaration for the C++ performance harness class.
+///
+// -----------------------------------------------------------------------------
+
+#ifndef HARNESS_H
+#define HARNESS_H
+
+#import <chrono>
+#import <functional>
+#import <iostream>
+#import <map>
+#import <string>
+#import <type_traits>
+#import <vector>
+
+/**
+ * Harness used for performance tests.
+ *
+ * The generator script will generate an extension to this class that adds a
+ * run() method, which the main.swift file calls.
+ */
+class Harness {
+public:
+  /**
+   * Creates a new harness that writes visualization output to the given
+   * output stream.
+   */
+  Harness(std::ostream* results_stream);
+
+  /**
+   * Runs the test harness. This function is generated by the script into
+   * _generated/Harness+Generated.cc.
+   */
+  void run();
+
+private:
+  /** Include fractional parts in milliseconds. */
+  typedef std::chrono::duration<double, std::milli> milliseconds_d;
+
+  /**
+   * Statistics representing the mean and standard deviation of all measured
+   * attempts.
+   */
+  struct Statistics {
+    double mean;
+    double stddev;
+  };
+
+  /** The output stream to which visualization results will be written. */
+  std::ostream* results_stream;
+
+  /** The number of times to loop the body of the run() method. */
+  int run_count;
+
+  /** The number of times to measure the function passed to measure(). */
+  int measurement_count;
+
+  /** The number of times to add values to repeated fields. */
+  int repeated_count;
+
+  /** The times taken by subtasks during each measured attempt. */
+  std::map<std::string, std::vector<milliseconds_d>> subtask_timings;
+
+  /** Times for the subtasks in the current attempt. */
+  std::map<std::string, std::chrono::steady_clock::duration> current_subtasks;
+
+  /**
+   * Measures the time it takes to execute the given function. The function is
+   * executed five times and the mean/standard deviation are computed.
+   */
+  template <typename Function>
+  void measure(const Function& func);
+
+  /**
+   * Measure an individual subtask whose timing will be printed separately
+   * from the main results.
+   */
+  template <typename Function>
+  typename std::result_of<Function()>::type measure_subtask(
+      const std::string& name, Function&& func);
+
+  /**
+   * Writes the given subtask's name and timings to the visualization log.
+   */
+  void write_to_log(const std::string& name,
+                    const std::vector<milliseconds_d>& timings) const;
+
+  /**
+   * Compute the mean and standard deviation of the given time points.
+   */
+  Statistics compute_statistics(
+      const std::vector<std::chrono::steady_clock::duration>& timings) const;
+};
+
+template <typename Function>
+void Harness::measure(const Function& func) {
+  using std::chrono::duration_cast;
+  using std::chrono::steady_clock;
+  using std::vector;
+
+  vector<steady_clock::duration> timings;
+  subtask_timings.clear();
+
+  // Do each measurement multiple times and collect the means and standard
+  // deviation to account for noise.
+  for (int attempt = 1; attempt <= measurement_count; attempt++) {
+    printf("Attempt %d, %d runs\n", attempt, run_count);
+    current_subtasks.clear();
+
+    auto start = steady_clock::now();
+    func();
+    auto end = steady_clock::now();
+    auto duration = end - start;
+    timings.push_back(duration);
+
+    for (const auto& entry : current_subtasks) {
+      auto millis = duration_cast<milliseconds_d>(entry.second);
+      printf("\"%s\" took %.3f ms\n",
+             entry.first.c_str(), millis.count());
+      subtask_timings[entry.first].push_back(millis);
+    }
+
+    auto millis = duration_cast<milliseconds_d>(duration);
+    printf("Total execution time: %.3f ms\n", millis.count());
+    printf("----\n");
+  }
+
+  for (const auto& entry : subtask_timings) {
+    write_to_log(entry.first, entry.second);
+  }
+
+  auto stats = compute_statistics(timings);
+  printf("mean = %.3f sec, stddev = %.3f sec\n", stats.mean, stats.stddev);
+}
+
+template <typename Function>
+typename std::result_of<Function()>::type Harness::measure_subtask(
+    const std::string& name, Function&& func) {
+  using std::chrono::steady_clock;
+
+  auto start = steady_clock::now();
+  auto result = func();
+  auto end = steady_clock::now();
+  auto diff = end - start;
+  current_subtasks[name] += diff;
+  return result;
+}
+
+#endif // HARNESS_H

Performance/Harness.swift

@@ -22,56 +22,87 @@ import Foundation
 /// run() method, which the main.swift file calls.
 class Harness {
 
+  /// The number of times to execute the block passed to measure().
+  var measurementCount = 10
+
   /// The number of times to loop the body of the run() method.
   var runCount = 100
 
   /// The number of times to call append() for repeated fields.
-  let repeatedCount: Int32 = 100
+  let repeatedCount: Int32 = 10
+
+  /// The times taken by subtasks during each measured attempt.
+  var subtaskTimings = [String: [TimeInterval]]()
+
+  /// Times for the subtasks in the current attempt.
+  var currentSubtasks = [String: TimeInterval]()
 
-  var subtasks = [String: TimeInterval]()
+  /// The file to which results should be written.
+  let resultsFile: FileHandle?
+
+  /// Creates a new harness that writes its statistics to the given file
+  /// (as well as to stdout).
+  init(resultsFile: FileHandle?) {
+    self.resultsFile = resultsFile
+  }
 
   /// Measures the time it takes to execute the given block. The block is
   /// executed five times and the mean/standard deviation are computed.
   func measure(block: () throws -> Void) {
     var timings = [TimeInterval]()
+    subtaskTimings.removeAll()
 
     do {
       // Do each measurement 5 times and collect the means and standard
       // deviation to account for noise.
-      for attempt in 1...5 {
+      for attempt in 1...measurementCount {
         print("Attempt \(attempt), \(runCount) runs:")
-        subtasks.removeAll()
+        currentSubtasks.removeAll()
 
         let start = Date()
         try block()
         let end = Date()
-        let diff = end.timeIntervalSince(start)
+        let diff = end.timeIntervalSince(start) * 1000
         timings.append(diff)
 
-        for (name, time) in subtasks {
-          print(String(format: "\"%@\" took %.3f sec", name, time))
+        for (name, time) in currentSubtasks {
+          print(String(format: "\"%@\" took %.3f ms", name, time))
+
+          if var timings = subtaskTimings[name] {
+            timings.append(time)
+            subtaskTimings[name] = timings
+          } else {
+            subtaskTimings[name] = [time]
+          }
         }
-        print(String(format: "Total execution time: %.3f sec\n", diff))
+        print(String(format: "Total execution time: %.3f ms\n", diff))
         print("----")
       }
     } catch let e {
       fatalError("Generated harness threw an error: \(e)")
     }
 
-    let (mean, stddev) = statistics(timings)
+    for (name, times) in subtaskTimings {
+      writeToLog("\"\(name)\": \(times),\n")
+    }
 
+    let (mean, stddev) = statistics(timings)
     let stats =
-        String(format: "mean = %.3f sec, stddev = %.3f sec\n", mean, stddev)
+        String(format: "mean = %.3f ms, stddev = %.3f ms\n", mean, stddev)
     print(stats)
   }
 
-  /// Measure an individual subtask whose timing will be printed separately from the main results.
-  func measureSubtask<Result>(_ name: String, block: () throws -> Result) rethrows -> Result {
+  /// Measure an individual subtask whose timing will be printed separately
+  /// from the main results.
+  func measureSubtask<Result>(
+    _ name: String,
+    block: () throws -> Result
+  ) rethrows -> Result {
     let start = Date()
     let result = try block()
     let end = Date()
-    let diff = end.timeIntervalSince(start)
-    subtasks[name] = (subtasks[name] ?? 0) + diff
+    let diff = end.timeIntervalSince(start) * 1000
+    currentSubtasks[name] = (currentSubtasks[name] ?? 0) + diff
     return result
   }
 
@@ -89,4 +120,13 @@ class Harness {
     let variance = sqsum / n - mean * mean
     return (mean: mean, stddev: sqrt(variance))
   }
+
+  /// Writes a string to the data results file that will be parsed by the
+  /// calling script to produce visualizations.
+  private func writeToLog(_ string: String) {
+    if let resultsFile = resultsFile {
+      let utf8 = Data(string.utf8)
+      resultsFile.write(utf8)
+    }
+  }
 }

Performance/css/harness-visualization.css

@@ -0,0 +1,29 @@
+body {
+  margin: 16px;
+}
+
+p.info-text {
+  padding: 16px;
+}
+
+h3 > small {
+  float: right;
+}
+
+table.numeric td {
+  text-align: right;
+}
+
+table.numeric th {
+  background-color: #eee;
+}
+
+table.numeric tfoot td {
+  font-size: 12px;
+  font-style: italic;
+}
+
+table.numeric td:first-child,
+table.numeric th:first-child {
+  text-align: left;
+}

Performance/harness-visualization.html

@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <title>Swift Protobuf Performance Harness Visualization</title>
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <script src="http://cdn.plot.ly/plotly-latest.min.js"></script>
+  <script src="http://cdnjs.cloudflare.com/ajax/libs/moment.js/2.17.1/moment.min.js"></script>
+  <script src="http://code.jquery.com/jquery-3.1.1.slim.min.js"></script>
+  <link rel="stylesheet" type="text/css" href="http://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css"/>
+  <link rel="stylesheet" type="text/css" href="css/harness-visualization.css"/>
+</head>
+<body>
+  <div id="container" class="container-fluid">
+    <div class="row">
+      <p class="info-text bg-info">Most recent sessions are listed first.</p>
+    </div>
+    <!-- Sessions will be inserted here. -->
+  </div>
+  <script src="_results/results.js"></script>
+  <script src="js/harness-visualization.js"></script>
+</body>
+</html>

Performance/js/harness-visualization.js

@@ -0,0 +1,262 @@
+(function() {
+  // The benchmarks that we want to display, in the order they should appear
+  // on the plot axis.
+  var benchmarks = [
+    'Populate fields',
+    'Encode binary', 'Decode binary',
+    'Encode JSON', 'Decode JSON',
+    'Encode text', 'Decode text',
+    'Test equality',
+  ];
+
+  // The languages we have harnesses for. The results for each language will
+  // appear as a series in the plot.
+  var languages = ['Swift', 'C++'];
+
+  // The harnessSize keys we want to print in the summary table, in the order
+  // they should be displayed.
+  var harnessSizeKeys = ['Unstripped', 'Stripped'];
+
+  // Common layout properties for the plot.
+  var layout = {
+    boxmode: 'group',
+    xaxis: {
+      showgrid: false,
+      showline: false,
+      autotick: true,
+      ticks: 'outside',
+    },
+    yaxis: {
+      title: 'Runtime (ms)',
+      autorange: true,
+    },
+    margin: {
+      l: 60,
+      r: 60,
+      t: 0,
+      b: 60,
+    },
+    font: {
+      family: 'Helvetica',
+    },
+    hovermode: 'closest',
+    legend: {
+      font: {
+        size: 12,
+      },
+      yanchor: 'middle',
+      xanchor: 'right',
+    },
+  };
+
+  // Creates and return a series for the given language's results in a session.
+  function createSeries(session, language) {
+    var x = [];
+    var y = [];
+
+    // The x-axis is categorical over the benchmark names. Adding the same
+    // benchmark multiple times will collapse all the points on the same
+    // vertical, which is what we want.
+    for (var i = 0; i < benchmarks.length; i++) {
+      var benchmark = benchmarks[i];
+      var timings = session[language][benchmark];
+      if (timings) {
+        for (var j = 0; j < timings.length; j++) {
+          x.push(benchmark.replace(" ", "<br>"));
+          y.push(timings[j]);
+        }
+      }
+    }
+
+    return {
+      name: language,
+      x: x,
+      y: y,
+      type: 'box',
+      boxpoints: 'all',
+      whiskerwidth: 0.5,
+      pointpos: 0,
+      jitter: 0.3,
+      mode: 'marker',
+      marker: {
+        symbol: 'circle',
+        size: 8,
+        opacity: 0.6,
+      },
+      line: {
+        width: 1,
+      },
+    };
+  }
+
+  // Computes and returns the median of the given array of values.
+  function median(values) {
+    values.sort();
+    var mid = Math.floor(values.length / 2);
+    if (values.length % 2) {
+      return values[mid];
+    } else {
+      return (values[mid - 1] + values[mid]) / 2.0;
+    }
+  }
+
+  // Populates a multiplier cell with the ratio between the given two values
+  // and sets its background color depending on the magnitude.
+  function populateMultiplierCell(cell, language, swiftValue, otherValue) {
+    if (language != 'Swift') {
+      var multiplier = swiftValue / otherValue;
+
+      if (multiplier < 1) {
+        cell.text('(<1x)');
+      } else {
+        cell.text('(' + multiplier.toFixed(0) + 'x)');
+      }
+
+      if (multiplier < 3) {
+        cssClass = 'bg-success';
+      } else if (multiplier < 10) {
+        cssClass = 'bg-warning';
+      } else {
+        cssClass = 'bg-danger';
+      }
+      cell.addClass(cssClass);
+    }
+  }
+
+  // Creates and returns the summary table displayed next to the chart for a
+  // given session.
+  function createSummaryTable(session) {
+    var table = $('<table></table>').addClass('table table-condensed numeric');
+    var tbody = $('<tbody></tbody>').appendTo(table);
+
+    // Insert the runtime stats.
+    var header = $('<tr></tr>').appendTo(table);
+    header.append($('<th>Median runtimes</th>'));
+    for (var j = 0; j < languages.length; j++) {
+      header.append($('<th></th>').text(languages[j]));
+      header.append($('<th></th>'));
+    }
+
+    for (var i = 0; i < benchmarks.length; i++) {
+      var benchmark = benchmarks[i];
+      var tr = $('<tr></tr>')
+      table.append(tr);
+      tr.append($('<td></td>').text(benchmark));
+
+      for (var j = 0; j < languages.length; j++) {
+        var language = languages[j];
+
+        var timings = session[language][benchmark];
+        if (timings) {
+          var med = median(timings);
+          var formattedMedian = med.toFixed(3) + '&nbsp;ms';
+          tr.append($('<td></td>').html(formattedMedian));
+
+          var multiplierCell = $('<td></td>').appendTo(tr);
+          var swiftMed = median(session['Swift'][benchmark]);
+          populateMultiplierCell(multiplierCell, language, swiftMed, med);
+        }
+      }
+    }
+
+    // Insert the binary size stats.
+    header = $('<tr></tr>').appendTo(table);
+    header.append($('<th>Harness size</th>'));
+    for (var j = 0; j < languages.length; j++) {
+      header.append($('<th></th>'));
+      header.append($('<th></th>'));
+    }
+
+    for (var i = 0; i < harnessSizeKeys.length; i++) {
+      var harnessSizeKey = harnessSizeKeys[i];
+      var tr = $('<tr></tr>')
+      table.append(tr);
+      tr.append($('<td></td>').text(harnessSizeKey));
+
+      for (var j = 0; j < languages.length; j++) {
+        var language = languages[j];
+
+        var size = session[language].harnessSize[harnessSizeKey];
+        var formattedSize = size.toLocaleString() + '&nbsp;b';
+        tr.append($('<td></td>').html(formattedSize));
+
+        var multiplierCell = $('<td></td>').appendTo(tr);
+        var swiftSize = session['Swift'].harnessSize[harnessSizeKey];
+        populateMultiplierCell(multiplierCell, language, swiftSize, size);
+      }
+    }
+
+    var tfoot = $('<tfoot></tfoot>').appendTo(table);
+    var footerRow = $('<tr></tr>').appendTo(tfoot);
+    var colspan = 2 * languages.length + 1;
+    var footerCell =
+        $('<td colspan="' + colspan + '"></td>').appendTo(footerRow);
+    footerCell.text('Multipliers indicate how much slower/larger the Swift ' +
+        'harness is compared to the other language.');
+
+    return table;
+  }
+
+  $(function() {
+    if (!window.sessions) {
+      return;
+    }
+
+    // Iterate the sessions in reverse order so that the most recent ones
+    // appear at the top. We create one chart for each session and tile them
+    // down the page.
+    for (var i = sessions.length - 1; i >= 0; i--) {
+      var session = sessions[i];
+      var allSeries = [];
+
+      formattedDate =
+          moment(new Date(session.date)).format('MMM Do h:mm:ss a');
+      var title = session.type;
+      var header = $('<h3></h3>').addClass('row').text(title);
+
+      var subtitle = 'Branch <tt>' + session.branch +
+          '</tt>, commit <tt>' + session.commit + '</tt>';
+      if (session.uncommitted_changes) {
+        subtitle += ' (with uncommited changes)';
+      }
+      subtitle += ', run on ' + formattedDate;
+
+      header.append($('<small></small>').html(subtitle));
+      $('#container').append('<hr>');
+      $('#container').append(header);
+
+      var id = 'chart' + i;
+      var row = $('<div></div>').addClass('row');
+      var chartColumn = $('<div></div>').addClass('col-md-9');
+      var tableColumn = $('<div></div>').addClass('col-md-3');
+
+      row.append(chartColumn);
+      row.append(tableColumn);
+      $('#container').append(row);
+
+      var chart = $('<div></div>').attr('id', id).addClass('chart');
+      chartColumn.append(chart);
+
+      for (var j = 0; j < languages.length; j++) {
+        var language = languages[j];
+        if (session[language]) {
+          var series = createSeries(session, language);
+          allSeries.push(series);
+        }
+      }
+
+      Plotly.newPlot(id, allSeries, layout, {
+        displayModeBar: false,
+      });
+
+      var table = createSummaryTable(session);
+      tableColumn.append(table);
+    }
+
+    window.onresize = function() {
+      $('.chart').each(function() {
+        Plotly.Plots.resize(this);
+      });
+    };
+  });
+})();

Performance/js/results.js.template

@@ -0,0 +1,3 @@
+sessions = [
+//NEW-DATA-HERE
+];

Performance/main.cc

@@ -0,0 +1,37 @@
+// Performance/main.cc - C++ performance harness entry point
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See http://swift.org/LICENSE.txt for license information
+// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+// -----------------------------------------------------------------------------
+///
+/// Entry point for the C++ performance harness.
+///
+// -----------------------------------------------------------------------------
+
+#include <fstream>
+
+#include "Harness.h"
+
+using std::ios_base;
+using std::ofstream;
+
+int main(int argc, char **argv) {
+  ofstream* results_stream = (argc > 1) ?
+      new ofstream(argv[1], ios_base::app) : nullptr;
+
+  Harness harness(results_stream);
+  harness.run();
+
+  if (results_stream) {
+    results_stream->close();
+    delete results_stream;
+  }
+
+  return 0;
+}

Performance/main.swift

@@ -14,5 +14,14 @@
 ///
 // -----------------------------------------------------------------------------
 
-let harness = Harness()
+import Foundation
+
+let args = CommandLine.arguments
+let resultsFile = args.count > 1 ?
+    FileHandle(forWritingAtPath: args[1]) : nil
+resultsFile?.seekToEndOfFile()
+
+let harness = Harness(resultsFile: resultsFile)
 harness.run()
+
+resultsFile?.closeFile()

Performance/perf_runner.sh

@@ -30,9 +30,15 @@
 
 set -eu
 
+readonly script_dir="$(dirname $0)"
+
+# Change this if your checkout of github.com/google/protobuf is in a different
+# location.
+readonly GOOGLE_PROTOBUF_CHECKOUT="$script_dir/../../protobuf"
+
 function usage() {
   cat >&2 <<EOF
-Usage: $0 [-p <true|false>] [-s2|-s3] <field count> <field type>
+Usage: $0 [-p <true|false>] [-s2|-s3] <field count> <field types...>
 
 Currently supported field types:
     int32, sint32, uint32, fixed32, sfixed32,
@@ -40,6 +46,10 @@ Currently supported field types:
     float, double, string,
     ...and repeated variants of the above.
 
+    Additionally, you can specify "all" to run the harness
+    multiple times with all of the (non-repeated) field types
+    listed above.
+
 Options:
     -p <true|false>: Adds a packed option to each field.
     -s[23]:          Generate proto2 or proto3 syntax. proto3 is
@@ -83,83 +93,62 @@ EOF
 }
 
 # ---------------------------------------------------------------
-# Functions for generating the Swift harness.
+# Functions for running harnesses and collecting results.
 
-function print_swift_set_field() {
-  num=$1
-  type=$2
+# Executes the test harness for a language under Instruments and concatenates
+# its results to the partial results file.
+function run_harness_and_concatenate_results() {
+  language="$1"
+  harness="$2"
+  partial_results="$3"
 
-  case "$type" in
-    repeated\ string)
-      echo "        for _ in 0..<repeatedCount {"
-      echo "          message.field$num.append(\"$((200+num))\")"
-      echo "        }"
-      ;;
-    repeated\ *)
-      echo "        for _ in 0..<repeatedCount {"
-      echo "          message.field$num.append($((200+num)))"
-      echo "        }"
-      ;;
-    string)
-      echo "        message.field$num = \"$((200+num))\""
-      ;;
-    *)
-      echo "        message.field$num = $((200+num))"
-      ;;
-  esac
-}
-
-function generate_perf_harness() {
-  cat >"$gen_harness_path" <<EOF
-extension Harness {
-  func run() {
-    measure {
-      // Loop enough times to get meaningfully large measurements.
-      for _ in 0..<runCount {
-        var message = PerfMessage()
-        measureSubtask("Populate message fields") {
-          populateFields(of: &message)
-        }
-
-        // Exercise binary serialization.
-        let data = try measureSubtask("Encode binary") {
-          return try message.serializeProtobuf()
-        }
-        message = try measureSubtask("Decode binary") {
-          return try PerfMessage(protobuf: data)
-        }
-
-        // Exercise JSON serialization.
-        let json = try measureSubtask("Encode JSON") {
-          return try message.serializeJSON()
-        }
-        let jsonDecodedMessage = try measureSubtask("Decode JSON") {
-          return try PerfMessage(json: json)
-        }
-
-        // Exercise equality.
-        measureSubtask("Test equality") {
-          guard message == jsonDecodedMessage else {
-            fatalError("Binary- and JSON-decoded messages were not equal!")
-          }
-        }
-      }
-    }
-  }
-
-  private func populateFields(of message: inout PerfMessage) {
+  cat >> "$partial_results" <<EOF
+    "$language": {
 EOF
 
-  for field_number in $(seq 1 "$field_count"); do
-    print_swift_set_field "$field_number" "$field_type" >>"$gen_harness_path"
-  done
-
-  cat >> "$gen_harness_path" <<EOF
-  }
-}
+  echo "Running $language test harness in Instruments..."
+  instruments -t "$script_dir/Protobuf" -D "$results_trace" \
+      "$harness" -e DYLD_LIBRARY_PATH "$script_dir/_generated" \
+      "$partial_results"
+
+  cp "$harness" "${harness}_stripped"
+  strip -u -r "${harness}_stripped"
+  unstripped_size=$(stat -f "%z" "$harness")
+  stripped_size=$(stat -f "%z" "${harness}_stripped")
+
+  echo "${language} harness size before stripping: $unstripped_size bytes"
+  echo "${language} harness size after stripping:  $stripped_size bytes"
+  echo
+
+  cat >> "$partial_results" <<EOF
+      harnessSize: {
+        "Unstripped": $unstripped_size,
+        "Stripped": $stripped_size,
+      },
+    },
 EOF
 }
 
+# Inserts the partial visualization results from all the languages tested into
+# the final results.js file.
+function insert_visualization_results() {
+  while IFS= read -r line
+  do
+    if [[ "$line" =~ ^//NEW-DATA-HERE$ ]]; then
+      cat "$partial_results"
+    fi
+    echo "$line"
+  done < "$results_js" > "${results_js}.new"
+
+  rm "$results_js"
+  mv "${results_js}.new" "$results_js"
+}
+
+# ---------------------------------------------------------------
+# Pull in language specific helpers.
+source "$script_dir/perf_runner_cpp.sh"
+source "$script_dir/perf_runner_swift.sh"
+
 # ---------------------------------------------------------------
 # Script main logic.
 
@@ -182,17 +171,20 @@ if [[ "$proto_syntax" != "2" ]] && [[ "$proto_syntax" != "3" ]]; then
   usage
 fi
 
-if [[ "$#" -ne 2 ]]; then
+if [[ "$#" -lt 2 ]]; then
   usage
 fi
 
-readonly field_count=$1
-readonly field_type=$2
-readonly script_dir="$(dirname $0)"
-
-# If the Instruments template has changed since the last run, copy it into the
-# user's template folder. (Would be nice if we could just run the template from
-# the local directory, but Instruments doesn't seem to support that.)
+readonly field_count="$1"; shift
+if [[ "$1" == "all" ]]; then
+  readonly requested_field_types=( \
+    int32 sint32 uint32 fixed32 sfixed32 \
+    int64 sint64 uint64 fixed64 sfixed64 \
+    float double string \
+  )
+else
+  readonly requested_field_types=( "$@" )
+fi
 
 # Make sure the runtime and plug-in are up to date first.
 ( cd "$script_dir/.." >/dev/null; swift build -c release )
@@ -205,44 +197,52 @@ cp "$script_dir/../.build/release/protoc-gen-swift" \
 mkdir -p "$script_dir/_generated"
 mkdir -p "$script_dir/_results"
 
-gen_message_path="$script_dir/_generated/message.proto"
-gen_harness_path="$script_dir/_generated/Harness+Generated.swift"
-results="$script_dir/_results/$field_count fields of $field_type"
-harness="$script_dir/_generated/harness"
-
-echo "Generating test proto with $field_count fields..."
-generate_test_proto "$field_count" "$field_type"
-
-echo "Generating test harness..."
-generate_perf_harness "$field_count" "$field_type"
-
-protoc --plugin="$script_dir/../.build/release/protoc-gen-swiftForPerf" \
-    --swiftForPerf_out=FileNaming=DropPath:"$script_dir/_generated" \
-    "$gen_message_path"
-
-echo "Building test harness..."
-time ( swiftc -O -target x86_64-apple-macosx10.10 \
-    -o "$harness" \
-    -I "$script_dir/../.build/release" \
-    -L "$script_dir/../.build/release" \
-    -lSwiftProtobuf \
-    "$gen_harness_path" \
-    "$script_dir/Harness.swift" \
-    "$script_dir/_generated/message.pb.swift" \
-    "$script_dir/main.swift" \
-)
-echo
-
-echo "Running test harness in Instruments..."
-instruments -t "$script_dir/Protobuf" -D "$results" "$harness"
-open "$results.trace"
-
-dylib="$script_dir/../.build/release/libSwiftProtobuf.dylib"
-echo "Dylib size before stripping: $(stat -f "%z" "$dylib") bytes"
-strip -u -r "$dylib"
-echo "Dylib size after stripping:  $(stat -f "%z" "$dylib") bytes"
-echo
-
-echo "Harness size before stripping: $(stat -f "%z" "$harness") bytes"
-strip -u -r "$harness"
-echo "Harness size after stripping:  $(stat -f "%z" "$harness") bytes"
+# If the visualization results file isn't there, copy it from the template so
+# that the harnesses can populate it.
+results_js="$script_dir/_results/results.js"
+if [[ ! -f "$results_js" ]]; then
+  cp "$script_dir/js/results.js.template" "$results_js"
+fi
+
+# Iterate over the requested field types and run the harnesses.
+for field_type in "${requested_field_types[@]}"; do
+  gen_message_path="$script_dir/_generated/message.proto"
+  results_trace="$script_dir/_results/$field_count fields of $field_type"
+
+  echo "Generating test proto with $field_count fields of type $field_type..."
+  generate_test_proto "$field_count" "$field_type"
+
+  protoc --plugin="$script_dir/../.build/release/protoc-gen-swiftForPerf" \
+      --swiftForPerf_out=FileNaming=DropPath:"$script_dir/_generated" \
+      --cpp_out="$script_dir" \
+      "$gen_message_path"
+
+  # Start a session.
+  partial_results="$script_dir/_results/partial.js"
+  cat > "$partial_results" <<EOF
+  {
+    date: "$(date -u +"%FT%T.000Z")",
+    type: "$field_count fields of type $field_type",
+    branch: "$(git rev-parse --abbrev-ref HEAD)",
+    commit: "$(git rev-parse HEAD)",
+    uncommitted_changes: $([[ -z $(git status -s) ]] && echo false || echo true),
+EOF
+
+  harness_swift="$script_dir/_generated/harness_swift"
+  run_swift_harness "$harness_swift"
+
+  harness_cpp="$script_dir/_generated/harness_cpp"
+  run_cpp_harness "$harness_cpp"
+
+  # Close out the session.
+  cat >> "$partial_results" <<EOF
+  },
+EOF
+
+  insert_visualization_results "$partial_results" "$results_js"
+
+  open -g "$results_trace.trace"
+done
+
+# Open the HTML once at the end.
+open -g "$script_dir/harness-visualization.html"

Performance/perf_runner_cpp.sh

@@ -0,0 +1,181 @@
+#!/bin/bash
+
+# SwiftProtobuf/Performance/perf_runner_cpp.sh - C++ test harness generator
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# -----------------------------------------------------------------------------
+#
+# Functions for generating the C++ harness.
+#
+# -----------------------------------------------------------------------------
+
+set -eu
+
+function print_cpp_set_field() {
+  num=$1
+  type=$2
+
+  case "$type" in
+    repeated\ string)
+      echo "        for (auto i = 0; i < repeated_count; i++) {"
+      echo "          message.add_field$num(\"$((200+num))\");"
+      echo "        }"
+      ;;
+    repeated\ *)
+      echo "        for (auto i = 0; i < repeated_count; i++) {"
+      echo "          message.add_field$num($((200+num)));"
+      echo "        }"
+      ;;
+    string)
+      echo "        message.set_field$num(\"$((200+num))\");"
+      ;;
+    *)
+      echo "        message.set_field$num($((200+num)));"
+      ;;
+  esac
+}
+
+function generate_cpp_harness() {
+  cat >"$gen_harness_path" <<EOF
+#include "Harness.h"
+#include "message.pb.h"
+
+#include <iostream>
+#include <google/protobuf/text_format.h>
+#include <google/protobuf/util/json_util.h>
+#include <google/protobuf/util/message_differencer.h>
+#include <google/protobuf/util/type_resolver_util.h>
+
+using google::protobuf::Descriptor;
+using google::protobuf::DescriptorPool;
+using google::protobuf::TextFormat;
+using google::protobuf::util::BinaryToJsonString;
+using google::protobuf::util::JsonToBinaryString;
+using google::protobuf::util::MessageDifferencer;
+using google::protobuf::util::NewTypeResolverForDescriptorPool;
+using google::protobuf::util::Status;
+using google::protobuf::util::TypeResolver;
+using std::cerr;
+using std::endl;
+using std::string;
+
+static const char kTypeUrlPrefix[] = "type.googleapis.com";
+
+static string GetTypeUrl(const Descriptor* message) {
+  return string(kTypeUrlPrefix) + "/" + message->full_name();
+}
+
+TypeResolver* type_resolver;
+string* type_url;
+
+static void populate_fields(PerfMessage& message);
+
+void Harness::run() {
+  GOOGLE_PROTOBUF_VERIFY_VERSION;
+
+  type_resolver = NewTypeResolverForDescriptorPool(
+      kTypeUrlPrefix, DescriptorPool::generated_pool());
+  type_url = new string(GetTypeUrl(PerfMessage::descriptor()));
+
+  measure([&]() {
+    // Loop enough times to get meaningfully large measurements.
+    for (auto i = 0; i < run_count; i++) {
+      auto message = PerfMessage();
+
+      measure_subtask("Populate fields", [&]() {
+        populate_fields(message);
+        // Dummy return value since void won't propagate.
+        return false;
+      });
+
+      // Exercise binary serialization.
+      auto data = measure_subtask("Encode binary", [&]() {
+        return message.SerializeAsString();
+      });
+      auto decoded_message = measure_subtask("Decode binary", [&]() {
+        auto result = PerfMessage();
+        result.ParseFromString(data);
+        return result;
+      });
+
+      // Exercise JSON serialization.
+      auto json = measure_subtask("Encode JSON", [&]() {
+        string out_json;
+        BinaryToJsonString(type_resolver, *type_url, data, &out_json);
+        return out_json;
+      });
+      auto decoded_binary = measure_subtask("Decode JSON", [&]() {
+        string out_binary;
+        JsonToBinaryString(type_resolver, *type_url, json, &out_binary);
+        return out_binary;
+      });
+
+      // Exercise text serialization.
+      auto text = measure_subtask("Encode text", [&]() {
+        string out_text;
+        TextFormat::PrintToString(message, &out_text);
+        return out_text;
+      });
+      measure_subtask("Decode text", [&]() {
+        auto result = PerfMessage();
+        TextFormat::ParseFromString(text, &result);
+        return result;
+      });
+
+      // Exercise equality.
+      measure_subtask("Test equality", [&]() {
+        return MessageDifferencer::Equals(message, decoded_message);
+      });
+    }
+  });
+
+  google::protobuf::ShutdownProtobufLibrary();
+}
+
+void populate_fields(PerfMessage& message) {
+EOF
+
+  for field_number in $(seq 1 "$field_count"); do
+    print_cpp_set_field "$field_number" "$field_type" >>"$gen_harness_path"
+  done
+
+  cat >> "$gen_harness_path" <<EOF
+}
+EOF
+}
+
+function run_cpp_harness() {
+  harness="$1"
+
+  echo "Generating C++ harness source..."
+  gen_harness_path="$script_dir/_generated/Harness+Generated.cc"
+  generate_cpp_harness "$field_count" "$field_type"
+
+  echo "Building C++ test harness..."
+  time ( g++ --std=c++11 -O \
+      -o "$harness" \
+      -I "$script_dir" \
+      -I "$GOOGLE_PROTOBUF_CHECKOUT/src" \
+      -L "$GOOGLE_PROTOBUF_CHECKOUT/src/.libs" \
+      -lprotobuf \
+      "$gen_harness_path" \
+      "$script_dir/Harness.cc" \
+      "$script_dir/_generated/message.pb.cc" \
+      "$script_dir/main.cc" \
+  )
+  echo
+
+  # Make sure the dylib is loadable from the harness if the user hasn't
+  # actually installed them.
+  cp "$GOOGLE_PROTOBUF_CHECKOUT"/src/.libs/libprotobuf.*.dylib \
+      "$script_dir/_generated"
+
+  run_harness_and_concatenate_results "C++" "$harness" "$partial_results"
+}

Performance/perf_runner_swift.sh

@@ -0,0 +1,132 @@
+#!/bin/bash
+
+# SwiftProtobuf/Performance/perf_runner_swift.sh - Swift test harness generator
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# -----------------------------------------------------------------------------
+#
+# Functions for generating the Swift harness.
+#
+# -----------------------------------------------------------------------------
+
+set -eu
+
+function print_swift_set_field() {
+  num=$1
+  type=$2
+
+  case "$type" in
+    repeated\ string)
+      echo "        for _ in 0..<repeatedCount {"
+      echo "          message.field$num.append(\"$((200+num))\")"
+      echo "        }"
+      ;;
+    repeated\ *)
+      echo "        for _ in 0..<repeatedCount {"
+      echo "          message.field$num.append($((200+num)))"
+      echo "        }"
+      ;;
+    string)
+      echo "        message.field$num = \"$((200+num))\""
+      ;;
+    *)
+      echo "        message.field$num = $((200+num))"
+      ;;
+  esac
+}
+
+function generate_swift_harness() {
+  cat >"$gen_harness_path" <<EOF
+extension Harness {
+  func run() {
+    measure {
+      // Loop enough times to get meaningfully large measurements.
+      for _ in 0..<runCount {
+        var message = PerfMessage()
+        measureSubtask("Populate fields") {
+          populateFields(of: &message)
+        }
+
+        // Exercise binary serialization.
+        let data = try measureSubtask("Encode binary") {
+          return try message.serializeProtobuf()
+        }
+        message = try measureSubtask("Decode binary") {
+          return try PerfMessage(protobuf: data)
+        }
+
+        // Exercise JSON serialization.
+        let json = try measureSubtask("Encode JSON") {
+          return try message.serializeJSON()
+        }
+        let jsonDecodedMessage = try measureSubtask("Decode JSON") {
+          return try PerfMessage(json: json)
+        }
+
+        // Exercise text serialization.
+        let text = try measureSubtask("Encode text") {
+          return try message.serializeText()
+        }
+        _ = try measureSubtask("Decode text") {
+          return try PerfMessage(text: text)
+        }
+
+        // Exercise equality.
+        measureSubtask("Test equality") {
+          guard message == jsonDecodedMessage else {
+            fatalError("Binary- and JSON-decoded messages were not equal!")
+          }
+        }
+      }
+    }
+  }
+
+  private func populateFields(of message: inout PerfMessage) {
+EOF
+
+  for field_number in $(seq 1 "$field_count"); do
+    print_swift_set_field "$field_number" "$field_type" >>"$gen_harness_path"
+  done
+
+  cat >> "$gen_harness_path" <<EOF
+  }
+}
+EOF
+}
+
+function run_swift_harness() {
+  harness="$1"
+
+  echo "Generating Swift harness source..."
+  gen_harness_path="$script_dir/_generated/Harness+Generated.swift"
+  generate_swift_harness "$field_count" "$field_type"
+
+  echo "Building Swift test harness..."
+  time ( swiftc -O -target x86_64-apple-macosx10.10 \
+      -o "$harness" \
+      -I "$script_dir/../.build/release" \
+      -L "$script_dir/../.build/release" \
+      -lSwiftProtobuf \
+      "$gen_harness_path" \
+      "$script_dir/Harness.swift" \
+      "$script_dir/_generated/message.pb.swift" \
+      "$script_dir/main.swift" \
+  )
+  echo
+
+  dylib="$script_dir/../.build/release/libSwiftProtobuf.dylib"
+  echo "Swift dylib size before stripping: $(stat -f "%z" "$dylib") bytes"
+  cp "$dylib" "${dylib}_stripped"
+  strip -u -r "${dylib}_stripped"
+  echo "Swift dylib size after stripping:  $(stat -f "%z" "${dylib}_stripped") bytes"
+  echo
+
+  run_harness_and_concatenate_results "Swift" "$harness_swift" "$partial_results"
+}