#pragma once #include #include #include #include #include #include namespace tf { /** @class: TaskQueue @tparam T data type (must be a pointer) @brief Lock-free unbounded single-producer multiple-consumer queue. This class implements the work stealing queue described in the paper, "Correct and Efficient Work-Stealing for Weak Memory Models," available at https://www.di.ens.fr/~zappa/readings/ppopp13.pdf. Only the queue owner can perform pop and push operations, while others can steal data from the queue. */ template class TaskQueue { static_assert(std::is_pointer_v, "T must be a pointer type"); struct Array { int64_t C; int64_t M; std::atomic* S; explicit Array(int64_t c) : C {c}, M {c-1}, S {new std::atomic[static_cast(C)]} { } ~Array() { delete [] S; } int64_t capacity() const noexcept { return C; } template void push(int64_t i, O&& o) noexcept { S[i & M].store(std::forward(o), std::memory_order_relaxed); } T pop(int64_t i) noexcept { return S[i & M].load(std::memory_order_relaxed); } Array* resize(int64_t b, int64_t t) { Array* ptr = new Array {2*C}; for(int64_t i=t; i!=b; ++i) { ptr->push(i, pop(i)); } return ptr; } }; std::atomic _top; std::atomic _bottom; std::atomic _array; std::vector _garbage; public: /** @brief constructs the queue with a given capacity @param capacity the capacity of the queue (must be power of 2) */ explicit TaskQueue(int64_t capacity = 1024); /** @brief destructs the queue */ ~TaskQueue(); /** @brief queries if the queue is empty at the time of this call */ bool empty() const noexcept; /** @brief queries the number of items at the time of this call */ size_t size() const noexcept; /** @brief queries the capacity of the queue */ int64_t capacity() const noexcept; /** @brief inserts an item to the queue Only the owner thread can insert an item to the queue. The operation can trigger the queue to resize its capacity if more space is required. @tparam O data type @param item the item to perfect-forward to the queue */ void push(T item); /** @brief pops out an item from the queue Only the owner thread can pop out an item from the queue. The return can be a nullptr if this operation failed (empty queue). */ T pop(); /** @brief steals an item from the queue Any threads can try to steal an item from the queue. The return can be a nullptr if this operation failed (not necessary empty). */ T steal(); }; // Constructor template TaskQueue::TaskQueue(int64_t c) { assert(c && (!(c & (c-1)))); _top.store(0, std::memory_order_relaxed); _bottom.store(0, std::memory_order_relaxed); _array.store(new Array{c}, std::memory_order_relaxed); _garbage.reserve(32); } // Destructor template TaskQueue::~TaskQueue() { for(auto a : _garbage) { delete a; } delete _array.load(); } // Function: empty template bool TaskQueue::empty() const noexcept { int64_t b = _bottom.load(std::memory_order_relaxed); int64_t t = _top.load(std::memory_order_relaxed); return b <= t; } // Function: size template size_t TaskQueue::size() const noexcept { int64_t b = _bottom.load(std::memory_order_relaxed); int64_t t = _top.load(std::memory_order_relaxed); return static_cast(b >= t ? b - t : 0); } // Function: push template void TaskQueue::push(T o) { int64_t b = _bottom.load(std::memory_order_relaxed); int64_t t = _top.load(std::memory_order_acquire); Array* a = _array.load(std::memory_order_relaxed); // queue is full if(a->capacity() - 1 < (b - t)) { Array* tmp = a->resize(b, t); _garbage.push_back(a); std::swap(a, tmp); _array.store(a, std::memory_order_relaxed); } a->push(b, o); std::atomic_thread_fence(std::memory_order_release); _bottom.store(b + 1, std::memory_order_relaxed); } // Function: pop template T TaskQueue::pop() { int64_t b = _bottom.load(std::memory_order_relaxed) - 1; Array* a = _array.load(std::memory_order_relaxed); _bottom.store(b, std::memory_order_relaxed); std::atomic_thread_fence(std::memory_order_seq_cst); int64_t t = _top.load(std::memory_order_relaxed); T item {nullptr}; if(t <= b) { item = a->pop(b); if(t == b) { // the last item just got stolen if(!_top.compare_exchange_strong(t, t+1, std::memory_order_seq_cst, std::memory_order_relaxed)) { item = nullptr; } _bottom.store(b + 1, std::memory_order_relaxed); } } else { _bottom.store(b + 1, std::memory_order_relaxed); } return item; } // Function: steal template T TaskQueue::steal() { int64_t t = _top.load(std::memory_order_acquire); std::atomic_thread_fence(std::memory_order_seq_cst); int64_t b = _bottom.load(std::memory_order_acquire); T item {nullptr}; if(t < b) { Array* a = _array.load(std::memory_order_consume); item = a->pop(t); if(!_top.compare_exchange_strong(t, t+1, std::memory_order_seq_cst, std::memory_order_relaxed)) { return nullptr; } } return item; } // Function: capacity template int64_t TaskQueue::capacity() const noexcept { return _array.load(std::memory_order_relaxed)->capacity(); } } // end of namespace tf -----------------------------------------------------