Commit f701dbad authored by Maged Michael's avatar Maged Michael Committed by Facebook Github Bot

Updated example and methodology for using DeterministicSchedule support for...

Updated example and methodology for using DeterministicSchedule support for auxiliary data and global invariants

Summary:
Depends on D3792669

Updating the test and methodology based on the experience with fine-grained testing dynamic MPMCQueue using DeterministicSchedule's support for auxiliary data and global invariants.

Updates D3675447

Reviewed By: djwatson

Differential Revision: D3794217

fbshipit-source-id: d2862895cb8dea120e758beeb24d6ae15191b013
parent 676035fe
...@@ -106,51 +106,86 @@ TEST(DeterministicSchedule, buggyAdd) { ...@@ -106,51 +106,86 @@ TEST(DeterministicSchedule, buggyAdd) {
} // for bug } // for bug
} // TEST } // TEST
/// Test support for auxiliary data and global invariants /// Test DSched support for auxiliary data and global invariants
/// ///
/// How to use DSched support for auxiliary data and global invariants
/// How to use DSched support for auxiliary data and global invariants: /// (Let Foo<T, Atom> be the template to be tested):
/// 1. Forward declare an annotated shared class /// 1. Add friend AnnotatedFoo<T> to Foo<T,Atom> (Typically, in Foo.h).
/// 2. Add the annotated shared class as a friend of the original class(es) /// 2. Define a class AuxData for whatever auxiliary data is needed
/// to be tested /// to maintain global knowledge of shared and private state.
/// 3. Define auxiliary data /// 3. Define:
/// 4. Define function(s) for updating auxiliary data to match shared updates /// static AuxData* aux_;
/// 5. Define the annotated shared class /// static FOLLY_TLS uint32_t tid_;
/// It supports an interface to the original class along with auxiliary /// 4. (Optional) Define gflags for command line options. E.g.:
/// functions for updating aux data, checking invariants, and/or logging /// DEFINE_int64(seed, 0, "Seed for random number generators");
/// It may have to duplicate the steps of multi-step operations in the /// 5. (Optionl) Define macros for mangement of auxiliary data. E.g.,
//// original code in order to manage aux data and check invariants after /// #define AUX_THR(x) (aux_->t_[tid_]->x)
/// shared accesses other than the first access in an opeeration /// 6. (Optional) Define macro for creating auxiliary actions. E.g.,
/// 6. Define function for checking global invariants and/or logging global /// #define AUX_ACT(act) \
/// state /// { \
/// 7. Define function generator(s) for function object(s) that update aux /// AUX_THR(func_) = __func__; \
/// data, check invariants, and/or log state /// AUX_THR(line_) = __LINE__; \
/// 8. Define TEST using anotated shared data, aux data, and aux functions /// AuxAct auxact([&](bool success) { if (success); act}); \
/// DeterministicSchedule::setAuxAct(auxact); \
/// }
/// [Note: Auxiliary actions must not contain any standard shared
/// accesses, or else deadlock will occur. Use the load_direct()
/// member function of DeterministicAtomic instead.]
/// 7. Define AnnotatedFoo<T> derived from Foo<T,DeterministicAtomic>.
/// 8. Define member functions in AnnotatedFoo to manage DSched::auxChk.
/// 9. Define member functions for logging and checkig global invariants.
/// 10. Define member functions for direct access to data members of Foo.
/// 11. (Optional) Add a member function dummyStep() to update
/// auxiliary data race-free when the next step is unknoown or
/// not conveniently accessible (e.g., in a different
/// library). The functions adds a dummy shared step to force
/// DSched to invoke the auxiliary action at a known point.This
/// is needed for now because DSched allows threads to run in
/// parallel between shared accesses. Hence, concurrent updates
/// of shared auxiliary data can be racy if executed outside
/// auxiliary actions. This may be obviated in the future if
/// DSched supports fully seriallized execution.
/// void dummyStep() {
/// DeterministicSchedule::beforeSharedAccess();
/// DeterministicSchedule::afterSharedAccess(true);
/// }
/// 12. Override member functions of Foo as needed in order to
/// annotate the code with auxiliary actions. [Note: There may be
/// a lot of duplication of Foo's code. Alternatively, Foo can be
/// annotated directly.]
/// 13. Define TEST using instances of AuxData and AnnotatedFoo.
/// 14. For debugging, iteratively add (as needed) auxiliary data,
/// global invariants, logging details, command line flags as
/// needed and selectively generate relevant logs to detect the
/// race condition shortly after it occurs.
///
/// In the following example Foo = AtomicCounter
using DSched = DeterministicSchedule; using DSched = DeterministicSchedule;
/** forward declaration of annotated shared class */ /** Forward declaration of annotated template */
class AnnotatedAtomicCounter; template <typename T>
struct AnnotatedAtomicCounter;
/** original shared class to be tested */ /** Original template to be tested */
template <typename T, template <typename> class Atom = std::atomic> template <typename T, template <typename> class Atom = std::atomic>
class AtomicCounter { class AtomicCounter {
friend AnnotatedAtomicCounter; /** Friend declaration to allow full access */
friend struct AnnotatedAtomicCounter<T>;
public: public:
explicit AtomicCounter(T val) : counter_(val) {} explicit AtomicCounter(T val) : counter_(val) {}
void inc() { void inc() {
counter_.fetch_add(1); this->counter_.fetch_add(1);
} }
void inc_bug() { void incBug() {
int newval = counter_.load() + 1; this->counter_.store(this->counter_.load() + 1);
counter_.store(newval);
} }
T load() { T load() {
return counter_.load(); return this->counter_.load();
} }
private: private:
...@@ -159,110 +194,176 @@ class AtomicCounter { ...@@ -159,110 +194,176 @@ class AtomicCounter {
/** auxiliary data */ /** auxiliary data */
struct AuxData { struct AuxData {
explicit AuxData(int nthr) { using T = int;
local_.resize(nthr, 0);
} /* General */
uint64_t step_ = {0};
uint64_t lastUpdate_ = {0};
struct PerThread {
/* General */
std::string func_;
int line_;
/* Custom */
T count_ = {0};
};
std::vector<PerThread> t_;
std::vector<int> local_; explicit AuxData(int nthr) : t_(nthr) {}
}; };
/** aux update function(s) */ static AuxData* aux_;
void auxUpdateAfterInc(int tid, AuxData& auxdata, bool success) { static FOLLY_TLS uint32_t tid_;
if (success) {
auxdata.local_[tid]++; /* Command line flags */
DEFINE_int64(seed, 0, "Seed for random number generators");
DEFINE_int64(max_steps, 1000000, "Max. number of shared steps for the test");
DEFINE_int64(num_reps, 1, "Number of test repetitions");
DEFINE_int64(num_ops, 1000, "Number of increments per repetition");
DEFINE_int64(liveness_thresh, 1000000, "Liveness threshold");
DEFINE_int64(log_begin, 0, "Step number to start logging. No logging if <= 0");
DEFINE_int64(log_length, 1000, "Length of step by step log (if log_begin > 0)");
DEFINE_int64(log_freq, 100000, "Log every so many steps");
DEFINE_int32(num_threads, 1, "Number of producers");
DEFINE_bool(bug, false, "Introduce bug");
/** Aux macros */
#define AUX_THR(x) (aux_->t_[tid_].x)
#define AUX_UPDATE() (aux_->lastUpdate_ = aux_->step_ + 1)
/** Macro for inline definition of auxiliary actions */
#define AUX_ACT(act) \
{ \
AUX_THR(func_) = __func__; \
AUX_THR(line_) = __LINE__; \
AuxAct auxfn( \
[&](bool success) { \
if (success); \
if (true) {act} \
} \
); \
DeterministicSchedule::setAuxAct(auxfn); \
} }
}
/** annotated shared class */ /** Alias for original class */
class AnnotatedAtomicCounter { template <typename T>
public: using Base = AtomicCounter<T, DeterministicAtomic>;
explicit AnnotatedAtomicCounter(int val) : shared_(val) {}
/** Annotated shared class */
template <typename T>
struct AnnotatedAtomicCounter : public Base<T> {
/** Manage DSched auxChk */
void setAuxChk() {
AuxChk auxfn(
[&](uint64_t step) {
auxLog(step);
auxCheck();
}
);
DeterministicSchedule::setAuxChk(auxfn);
}
void inc(AuxAct& auxfn) { void clearAuxChk() {
DSched::setAuxAct(auxfn); DeterministicSchedule::clearAuxChk();
/* calls the fine-grained original */
shared_.inc();
} }
void inc_bug(AuxAct auxfn) { /** Aux log function */
/* duplicates the steps of the multi-access original in order to void auxLog(uint64_t step) {
* annotate the second access */ if (aux_->step_ == 0) {
int newval = shared_.counter_.load() + 1; aux_->lastUpdate_ = step;
DSched::setAuxAct(auxfn); }
shared_.counter_.store(newval); aux_->step_ = step;
if (step > (uint64_t)FLAGS_max_steps) {
exit(0);
}
bool doLog =
(((FLAGS_log_begin > 0) && (step >= (uint64_t)FLAGS_log_begin) &&
(step <= (uint64_t)FLAGS_log_begin + FLAGS_log_length)) ||
((step % FLAGS_log_freq) == 0));
if (doLog) {
doAuxLog(step);
}
} }
int load_direct() { void doAuxLog(uint64_t step) {
return shared_.counter_.load_direct(); std::stringstream ss;
/* General */
ss << step << " - " << aux_->lastUpdate_ << " --";
/* Shared */
ss << " counter =" << this->counter_.load_direct();
/* Thread */
ss << " -- t" << tid_ << " " << AUX_THR(func_) << ":" << AUX_THR(line_);
ss << " count[" << tid_ << "] = " << AUX_THR(count_);
/* Output */
std::cerr << ss.str() << std::endl;
} }
private: void auxCheck() {
AtomicCounter<int, DeterministicAtomic> shared_; /* Liveness */
}; CHECK_LT(aux_->step_, aux_->lastUpdate_ + FLAGS_liveness_thresh);
/* Safety */
int sum = {0};
for (auto& t : aux_->t_) {
sum += t.count_;
}
CHECK_EQ(this->counter_.load_direct(), sum);
}
using Annotated = AnnotatedAtomicCounter; /* Direct access without going through DSched */
T loadDirect() {
return this->counter_.load_direct();
}
/* Constructor -- calls original constructor */
AnnotatedAtomicCounter(int val) : Base<T>(val) {}
/* Overloads of original member functions (as needed) */
/** aux log & check function */ void inc() {
void auxCheck(int tid, Annotated& annotated, AuxData& auxdata) { AUX_ACT({ ++AUX_THR(count_); });
/* read shared data */ this->counter_.fetch_add(1);
int val = annotated.load_direct();
/* read auxiliary data */
int sum = 0;
for (int v : auxdata.local_) {
sum += v;
} }
/* log state */
VLOG(2) << "tid " << tid << " -- shared counter= " << val void incBug() {
<< " -- sum increments= " << sum; AUX_ACT({});
/* check invariant */ T newval = this->counter_.load() + 1;
if (val != sum) { AUX_ACT({ ++AUX_THR(count_); });
LOG(ERROR) << "counter=(" << val << ") expected(" << sum << ")"; this->counter_.store(newval);
CHECK(false);
} }
} };
/** function generator(s) */ using Annotated = AnnotatedAtomicCounter<int>;
AuxAct auxAfterInc(int tid, Annotated& annotated, AuxData& auxdata) {
return [&annotated, &auxdata, tid](bool success) {
auxUpdateAfterInc(tid, auxdata, success);
auxCheck(tid, annotated, auxdata);
};
}
DEFINE_bool(bug, false, "Introduce bug"); TEST(DeterministicSchedule, global_invariants) {
DEFINE_int64(seed, 0, "Seed for random number generator"); CHECK_GT(FLAGS_num_threads, 0);
DEFINE_int32(num_threads, 2, "Number of threads");
DEFINE_int32(num_iterations, 10, "Number of iterations"); DSched sched(DSched::uniform(FLAGS_seed));
for (int i = 0; i < FLAGS_num_reps; ++i) {
TEST(DSchedCustom, atomic_add) { aux_ = new AuxData(FLAGS_num_threads);
bool bug = FLAGS_bug; Annotated annotated(0);
long seed = FLAGS_seed; annotated.setAuxChk();
int nthr = FLAGS_num_threads;
int niter = FLAGS_num_iterations; std::vector<std::thread> threads(FLAGS_num_threads);
for (int tid = 0; tid < FLAGS_num_threads; ++tid) {
CHECK_GT(nthr, 0); threads[tid] = DSched::thread([&, tid]() {
tid_ = tid;
Annotated annotated(0); for (int j = tid; j < FLAGS_num_ops; j += FLAGS_num_threads) {
AuxData auxdata(nthr); (FLAGS_bug) ? annotated.incBug() : annotated.inc();
DSched sched(DSched::uniform(seed));
std::vector<std::thread> threads(nthr);
for (int tid = 0; tid < nthr; ++tid) {
threads[tid] = DSched::thread([&, tid]() {
AuxAct auxfn = auxAfterInc(tid, annotated, auxdata);
for (int i = 0; i < niter; ++i) {
if (bug && (tid == 0) && (i % 10 == 0)) {
annotated.inc_bug(auxfn);
} else {
annotated.inc(auxfn);
} }
} });
}); }
} for (auto& t : threads) {
for (auto& t : threads) { DSched::join(t);
DSched::join(t); }
std::cerr << "====== rep " << i << " completed in step " << aux_->step_
<< std::endl;
annotated.doAuxLog(aux_->step_);
std::cerr << std::endl;
EXPECT_EQ(annotated.loadDirect(), FLAGS_num_ops);
annotated.clearAuxChk();
delete aux_;
} }
EXPECT_EQ(annotated.load_direct(), nthr * niter);
} }
int main(int argc, char** argv) { int main(int argc, char** argv) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment