bslmt_throughputbenchmark.h

Go to the documentation of this file.

/// @file bslmt_throughputbenchmark.h
///
/// The content of this file has been pre-processed for Doxygen.
///
 
 
// bslmt_throughputbenchmark.h                                        -*-C++-*-
 
#ifndef INCLUDED_BSLMT_THROUGHPUTBENCHMARK
#define INCLUDED_BSLMT_THROUGHPUTBENCHMARK
 
#include <bsls_ident.h>
BSLS_IDENT("$Id: $")
 
/// @defgroup bslmt_throughputbenchmark bslmt_throughputbenchmark
/// @brief  Provide a performance test harness for multi-threaded components.
/// @addtogroup bsl
/// @{
/// @addtogroup bslmt
/// @{
/// @addtogroup bslmt_throughputbenchmark
/// @{
///
/// <h1> Outline </h1>
/// * <a href="#bslmt_throughputbenchmark-purpose"> Purpose</a>
/// * <a href="#bslmt_throughputbenchmark-classes"> Classes </a>
/// * <a href="#bslmt_throughputbenchmark-description"> Description </a>
///   * <a href="#bslmt_throughputbenchmark-structure-of-a-test"> Structure of a Test </a>
///   * <a href="#bslmt_throughputbenchmark-usage"> Usage </a>
///     * <a href="#bslmt_throughputbenchmark-example-1-test-performance-of-bsl-queue-int"> Example 1: Test Performance of bsl::queue<int> </a>
///
/// # Purpose {#bslmt_throughputbenchmark-purpose}
/// Provide a performance test harness for multi-threaded components.
///
/// # Classes {#bslmt_throughputbenchmark-classes}
///
/// -  bslmt::ThroughputBenchmark: multi-threaded performance test harness
///
/// # Description {#bslmt_throughputbenchmark-description}
/// This component defines a mechanism,
/// `bslmt::ThroughputBenchmark`, that provides performance testing for multi-
/// threaded components.  The results are loaded into a
/// `bslmt::ThroughputBenchmarkResult` object, which provides access to counts
/// of the work done by each thread, thread group, and sample, divided by the
/// number of actual seconds of execution.
///
/// ## Structure of a Test {#bslmt_throughputbenchmark-structure-of-a-test}
///
///
/// A test is composed from one or more thread groups, each running one or more
/// threads.  Each thread in a thread group executes a thread function, with a
/// simulated work load executing between subsequent calls to the thread
/// function.  To provide reliability, the test is executed multiple times.  A
/// single execution of a test is referred to as a "sample execution" and its
/// result referred to as a "sample".  To support fine tuning of the test, it is
/// possible to provide initialize and cleanup functions for a sample and / or a
/// thread.
///
/// ## Usage {#bslmt_throughputbenchmark-usage}
///
///
/// This section illustrates intended use of this component.
///
/// ### Example 1: Test Performance of bsl::queue<int> {#bslmt_throughputbenchmark-example-1-test-performance-of-bsl-queue-int}
///
///
/// In the following example we test the throughput of a `bsl::queue<int>` in a
/// multi-threaded environment, where multiple "producer" threads are pushing
/// elements, and multiple "consumer" threads are popping these elements.
///
/// First, we define a global queue, a mutex to protect this queue, and a
/// semaphore for a "pop" operation to block on:
/// @code
/// bsl::queue<int>  myQueue;
/// bslmt::Mutex     myMutex;
/// bslmt::Semaphore mySem;
/// @endcode
/// Next, we define a counter value we push in:
/// @code
/// int              counterValue = 0;
/// @endcode
/// Then, we define simple push and pop functions that manipulate this queue:
/// @code
/// /// Push an element into `myQueue`, using the specified `threadIndex`.
/// void myPush(int threadIndex)
/// {
///     bslmt::LockGuard<bslmt::Mutex> guard(&myMutex);
///     myQueue.push(1000000 * threadIndex + counterValue++);
///     mySem.post();
/// }
///
/// /// Pop an element from `myQueue`.
/// void myPop(int)
/// {
///     mySem.wait();
///     bslmt::LockGuard<bslmt::Mutex> guard(&myMutex);
///     myQueue.pop();
/// }
/// @endcode
/// Next, we define a thread "cleanup" function for the push thread group, which
/// pushes a couple of extra elements to make sure that the pop thread group
/// will not hang on an empty queue:
/// @code
/// /// Cleanup function.
/// void myCleanup()
/// {
///     bslmt::LockGuard<bslmt::Mutex> guard(&myMutex);
///     for (int i = 0; i < 10; ++i) {
///         myQueue.push(counterValue++);
///         mySem.post();
///     }
/// }
/// @endcode
/// Then, we create a `bslmt::ThroughputBenchmark` object and add push and pop
/// thread groups, each with 2 threads and a work load (arithmetic operations to
/// consume an amount of time) of 100:
/// @code
/// bslmt::ThroughputBenchmark myBench;
/// myBench.addThreadGroup(
///                     myPush,
///                     2,
///                     100,
///                     bslmt::ThroughputBenchmark::InitializeThreadFunction(),
///                     myCleanup);
/// const int consumerGroupIdx = myBench.addThreadGroup(myPop, 2, 100);
/// @endcode
/// Now, we create a `bslmt::ThroughputBenchmarkResult` object to contain the
/// result, and call `execute` to run the benchmark for 500 millseconds 10
/// times:
/// @code
/// bslmt::ThroughputBenchmarkResult myResult;
/// myBench.execute(&myResult, 500, 10);
/// @endcode
/// Finally, we print the median of the throughput of the consumer thread group.
/// @code
/// double median;
/// myResult.getMedian(&median, consumerGroupIdx);
/// bsl::cout << "Throughput:" << median << "\n";
/// @endcode
/// @}
/** @} */
/** @} */
 
/** @addtogroup bsl
 * @{
 */
/** @addtogroup bslmt
 * @{
 */
/** @addtogroup bslmt_throughputbenchmark
 * @{
 */
 
#include <bslscm_version.h>
 
#include <bslmt_barrier.h>
#include <bslmt_throughputbenchmarkresult.h>
 
#include <bslma_allocator.h>
 
#include <bslmf_nestedtraitdeclaration.h>
 
#include <bsls_assert.h>
#include <bsls_atomic.h>
#include <bsls_timeinterval.h>
#include <bsls_types.h>
 
#include <bsl_functional.h>
#include <bsl_vector.h>
 
 
namespace bslmt {
 
class ThroughputBenchmark_TestUtil;
 
                        // =========================
                        // class ThroughputBenchmark
                        // =========================
 
/// This class is a mechanism that provides performance testing for multi-
/// threaded components.  It allows running different thread functions at
/// the same time, and simulates a work load between subsequent calls to the
/// tested thread functions.  The results are loaded into a
/// `bslmt::ThroughputBenchmarkResult` object, which provides access to
/// counts of the work done by each thread, thread group, and sample,
/// divided by the number of actual seconds of execution.
///
/// See @ref bslmt_throughputbenchmark 
class ThroughputBenchmark {
 
  public:
    // PUBLIC TYPES
 
    /// An alias to a function meeting the following contract:
    /// @code
    /// /// Run the main part of the benchmark having the specified /// `threadIndex`.  
    /// 
    /// \pre The behavior is undefined   unless `threadIndex` is
    /// /// in the range `[0, numThreadsInGroup)`, where `numThreadsInGroup` is
    /// /// the number of threads in a thread group for the associated
    /// /// throughput benchmark.
    /// void runTest(int threadIndex);
    /// @endcode
    typedef bsl::function<void(int)> RunFunction;
 
    /// An alias to a function meeting the following contract:
    /// @code
    /// /// Initialize the sample run.  If the specified `isFirst` is `true`,
    /// /// this is the first sample run.
    /// void initializeSample(bool isFirst);
    /// @endcode
    typedef bsl::function<void(bool)> InitializeSampleFunction;
 
    /// An alias to a function meeting the following contract:
    /// @code
    /// /// Clean up at the end of the sample run, before threads have been
    /// /// joined.  If the specified `isLast` is `true`, this is the last
    /// /// sample run.
    /// void shutdownSample(bool isLast);
    /// @endcode
    typedef bsl::function<void(bool)> ShutdownSampleFunction;
 
    /// An alias to a function meeting the following contract:
    /// @code
    /// /// Clean up after the sample run.  If the specified `isLast` is
    /// /// `true`, this is the last sample run.
    /// void cleanupSample(bool isLast);
    /// @endcode
    typedef bsl::function<void(bool)> CleanupSampleFunction;
 
    /// An alias to a function meeting the following contract:
    /// @code
    /// /// Initialize each thread in a sample run.
    /// void initializeThread();
    /// @endcode
    typedef bsl::function<void()> InitializeThreadFunction;
 
    /// An alias to a function meeting the following contract:
    /// @code
    /// /// Clean up after each thread in a sample run.
    /// void cleanupThread();
    /// @endcode
    typedef bsl::function<void()> CleanupThreadFunction;
 
    /// Data used by a thread group
    ///
    /// See @ref bslmt_throughputbenchmark 
    struct ThreadGroup {
 
        // PUBLIC DATA
        RunFunction               d_func;         // test function to run
 
        int                       d_numThreads;   // number of threads in the
                                                  // thread group
 
        bsls::Types::Int64        d_amount;       // amount of busy work to
                                                  // perform between calls to
                                                  // 'd_func'
 
        InitializeThreadFunction  d_initialize;   // initialize function per
                                                  // thread
 
        CleanupThreadFunction     d_cleanup;      // cleanup function per
                                                  // thread
    };
 
  private:
    // CLASS DATA
    static bsls::AtomicUint   s_antiOptimization; // Used by 'busyWork' to
                                                  // prevent optimization.
 
    // DATA
    bsl::vector<ThreadGroup>  d_threadGroups;     // Data kept for each thread
                                                  // group added.
 
    bsls::AtomicInt           d_state;            // This is how a test thread
                                                  // knows it has to exit.  It
                                                  // starts as 0, and exits
                                                  // when is set to 1.
 
    // FRIENDS
    friend class ThroughputBenchmark_WorkFunction;
    friend class ThroughputBenchmark_TestUtil;
 
  private:
    // NOT IMPLEMENTED
    ThroughputBenchmark(const ThroughputBenchmark&);
    ThroughputBenchmark& operator=(const ThroughputBenchmark&);
 
    // PRIVATE ACCESSORS
 
    /// Return `true` if the test should continue to run, and `false`
    /// otherwise.
    bool isRunState() const;
 
  public:
    // TRAITS
    BSLMF_NESTED_TRAIT_DECLARATION(ThroughputBenchmark,
                                   bslma::UsesBslmaAllocator);
 
    // CLASS METHODS
 
    /// Return the value calculated by `busyWork`.  
    /// \note Note that  this method is
    /// provided to prevent the compiler from optimizing the simulated workload
    /// away.
    static unsigned int antiOptimization();
 
    /// Perform arithmetic operations to consume an amount of time in linear relation to the specified `busyWorkAmount`.  
    /// 
    /// \note Note that   the duration of
    /// `busyWork` invoked with a particular `busyWorkAmount` will vary with
    /// system load.
    static void busyWork(bsls::Types::Int64 busyWorkAmount);
 
    /// Return an estimate of the work amount so that `busyWork` invoked with
    /// the returned work amount executes, approximately, for the specified `duration`.  
    /// 
    /// \note Note that   this estimate varies with system load.
    static bsls::Types::Int64 estimateBusyWorkAmount(
                                                  bsls::TimeInterval duration);
 
    // CREATORS
 
    /// Create an empty `ThroughputBenchmark` object.  Optionally specify a
    /// `basicAllocator` used to supply memory.  If `basicAllocator` is 0, the
    /// currently installed default allocator is used.
    explicit ThroughputBenchmark(bslma::Allocator *basicAllocator = 0);
 
    // MANIPULATORS
 
    /// Create a set of threads, with cardinality the specified `numThreads`,
    /// that will repeatedly execute the specified `runFunction` followed by
    /// the specified `busyWork`, with the specified `busyWorkAmount` as its
    /// argument.  Return the index for the thread group.  Optionally specify
    /// `initializeFunctor`, which is run at the beginning of each thread of
    /// the sample and accepts a boolean flag `isFirst`, that is set to `true`
    /// on the first sample, and `false` otherwise.  Optionally specify
    /// `cleanupFunctor`, which is run at the end of each thread of the sample
    /// and accepts a boolean flag `isLast`, that is set to `true` on the last
    /// sample, and `false` otherwise.  Return an id for the added thread group.  
    /// 
    /// \pre The behavior is undefined   unless `0 < numThreads` and
    /// `0 <= busyWorkAmount`.
    int addThreadGroup(const RunFunction& runFunction,
                       int                numThreads,
                       bsls::Types::Int64 busyWorkAmount);
    int addThreadGroup(const RunFunction&              runFunction,
                       int                             numThreads,
                       bsls::Types::Int64              busyWorkAmount,
                       const InitializeThreadFunction& initializeFunctor,
                       const CleanupThreadFunction&    cleanupFunctor);
 
    /// Run the tests previously added with calls to the `addThreadGroup`
    /// method.  The tests are run for the specified `numSamples` times.  Each
    /// sample is run for the specified `millisecondsPerSample` duration.  The
    /// results are stored in the specified `result` object.  Optionally
    /// specify `initializeFunctor`, which is run at the beginning of the
    /// sample and accepts a boolean flag `isFirst`, that is set to `true` on
    /// the first sample, and `false` otherwise.  Optionally specify
    /// `shutdownFunctor`, which is run at the end of each sample before
    /// threads have been joined, and accepts a boolean flag `isLast`, that is
    /// set to `true` on the last sample, and `false` otherwise.  Optionally
    /// specify `cleanupFunctor`, which is run at the end of each sample after
    /// threads have been joined, and accepts a boolean flag `isLast`, that is
    /// set to `true` on the last sample, and `false` otherwise.  
    /// 
    /// \pre The behavior is undefined   unless `0 < millisecondsPerSample`, `0 < numSamples`, and
    /// `0 < numThreadGroups()`.  Also see @ref bslmt_throughputbenchmark-structure-of-a-test .
    void execute(ThroughputBenchmarkResult       *result,
                 int                              millisecondsPerSample,
                 int                              numSamples);
    void execute(ThroughputBenchmarkResult       *result,
                 int                              millisecondsPerSample,
                 int                              numSamples,
                 const InitializeSampleFunction&  initializeFunctor,
                 const ShutdownSampleFunction&    shutdownFunctor,
                 const CleanupSampleFunction&     cleanupFunctor);
 
    // ACCESSORS
 
    /// Return the total number of threads.
    int numThreads() const;
 
    /// Return the number of thread groups.
    int numThreadGroups() const;
 
    /// Return the number of threads in the specified `threadGroupIndex`. 
    /// 
    /// \pre The behavior is undefined   unless
    /// `0 <= threadGroupIndex < numThreadGroups()`.
    int numThreadsInGroup(int threadGroupIndex) const;
 
                                  // Aspects
 
    /// Return the allocator used by this object.
    bslma::Allocator *allocator() const;
 
};
 
                   // ===================================
                   // struct ThroughputBenchmark_WorkData
                   // ===================================
 
/// Data transferred to ThroughputBenchmark_WorkFunction.
///
/// See @ref bslmt_throughputbenchmark 
struct ThroughputBenchmark_WorkData {
 
    // PUBLIC DATA
    ThroughputBenchmark::RunFunction              d_func;
                                                    // test function to run
 
    bsls::Types::Int64                            d_amount;
                                                    // busy work amount
 
    ThroughputBenchmark::InitializeThreadFunction d_initialize;
                                                    // initialize function per
                                                    // thread
 
    ThroughputBenchmark::CleanupThreadFunction    d_cleanup;
                                                    // cleanup function per
                                                    // thread
 
    ThroughputBenchmark                          *d_bench_p;
                                                    // exposes the "this"
                                                    // pointer of the benchmark
                                                    // to the work thread
 
    int                                           d_threadIndex;
                                                    // thread index 0, 1, 2,
                                                    // ... that is provided to
                                                    // the thread to
                                                    // differentiate it if so
                                                    // desired
 
    bslmt::Barrier                               *d_barrier_p;
                                                    // trigger start for
                                                    // threads to start
                                                    // processing at the same
                                                    // time
 
    bsls::Types::Int64                            d_actualNanos;
                                                    // number of nanoseconds
                                                    // that the thread actually
                                                    // ran
 
    bsls::Types::Int64                            d_count;
                                                    // number of items
                                                    // processed by this thread
};
 
                  // ======================================
                  // class ThroughputBenchmark_WorkFunction
                  // ======================================
 
/// This class is the work function functor, being called for each work
/// thread.
///
/// See @ref bslmt_throughputbenchmark 
class ThroughputBenchmark_WorkFunction {
 
  private:
    // DATA
    ThroughputBenchmark_WorkData& d_data;
 
  public:
    // CREATORS
 
    /// Create a `ThroughputBenchmark_WorkFunction` object with the
    /// specified `data` argument.
    explicit ThroughputBenchmark_WorkFunction(
                                           ThroughputBenchmark_WorkData& data);
 
    /// Destroy this object.
     ~ThroughputBenchmark_WorkFunction() = default;
 
    // MANIPULATORS
 
    /// Work function being run on the thread.
    void operator()();
};
 
                    // ==================================
                    // class ThroughputBenchmark_TestUtil
                    // ==================================
 
/// This class implements a test utility that gives the test driver access
/// to the unexposed data members of `ThroughputBenchmark`.
///
/// See @ref bslmt_throughputbenchmark 
class ThroughputBenchmark_TestUtil {
 
    // DATA
    ThroughputBenchmark& d_data;
 
  public:
    // CREATORS
 
    /// Create a `ThroughputBenchmark_TestUtil` object to test contents of
    /// the specified `data`.
    explicit ThroughputBenchmark_TestUtil(ThroughputBenchmark& data);
 
    /// Destroy this object.
     ~ThroughputBenchmark_TestUtil() = default;
 
    // MANIPULATORS
 
    /// Return a reference providing modifiable access to the `d_state` data
    /// member of `ThroughputBenchmark`.
    bsls::AtomicInt& state();
 
    /// Return a reference providing modifiable access to the
    /// `d_threadGroups` data member of `ThroughputBenchmark`.
    bsl::vector<ThroughputBenchmark::ThreadGroup>& threadGroups();
};
 
// ============================================================================
//                             INLINE DEFINITIONS
// ============================================================================
 
                        // -------------------------
                        // class ThroughputBenchmark
                        // -------------------------
 
// PRIVATE ACCESSORS
inline
bool ThroughputBenchmark::isRunState() const
{
    return d_state.loadAcquire() == 0;
}
 
// ACCESSORS
inline
int ThroughputBenchmark::numThreads() const
{
    if (0 == d_threadGroups.size()) {
        return 0;                                                     // RETURN
    }
 
    int numThreads = d_threadGroups[0].d_numThreads;
    for (int i = 1; i < numThreadGroups(); ++i) {
        numThreads += d_threadGroups[i].d_numThreads;
    }
    return numThreads;
}
 
inline
int ThroughputBenchmark::numThreadGroups() const
{
    return static_cast<int>(d_threadGroups.size());
}
 
inline
int ThroughputBenchmark::numThreadsInGroup(int threadGroupIndex) const
{
    BSLS_ASSERT(0                 <= threadGroupIndex);
    BSLS_ASSERT(numThreadGroups() >  threadGroupIndex);
 
    return d_threadGroups[threadGroupIndex].d_numThreads;
}
 
                                  // Aspects
 
inline
bslma::Allocator* ThroughputBenchmark::allocator() const
{
    return d_threadGroups.get_allocator().mechanism();
}
 
                  // --------------------------------------
                  // class ThroughputBenchmark_WorkFunction
                  // --------------------------------------
 
// CREATORS
inline
ThroughputBenchmark_WorkFunction::ThroughputBenchmark_WorkFunction(
                                            ThroughputBenchmark_WorkData& data)
: d_data(data)
{
}
 
                    // ----------------------------------
                    // class ThroughputBenchmark_TestUtil
                    // ----------------------------------
 
// CREATORS
inline
ThroughputBenchmark_TestUtil::ThroughputBenchmark_TestUtil(
                                                     ThroughputBenchmark& data)
: d_data(data)
{
}
 
// MANIPULATORS
inline
bsls::AtomicInt& ThroughputBenchmark_TestUtil::state()
{
    return d_data.d_state;
}
 
inline
bsl::vector<ThroughputBenchmark::ThreadGroup>&
                                   ThroughputBenchmark_TestUtil::threadGroups()
{
    return d_data.d_threadGroups;
}
 
}  // close package namespace
 
 
#endif
 
// ----------------------------------------------------------------------------
// Copyright 2019 Bloomberg Finance L.P.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ----------------------------- END-OF-FILE ----------------------------------
 
/** @} */
/** @} */
/** @} */