//
// Copyright 2012 Francisco Jerez
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
// OTHER DEALINGS IN THE SOFTWARE.
//
#include "core/kernel.hpp"
#include "core/resource.hpp"
#include "util/factor.hpp"
#include "util/u_math.h"
#include "pipe/p_context.h"
using namespace clover;
kernel::kernel(clover::program &prog, const std::string &name,
const std::vector<module::argument> &margs) :
program(prog), _name(name), exec(*this),
program_ref(prog._kernel_ref_counter) {
for (auto &marg : margs) {
if (marg.semantic == module::argument::general)
_args.emplace_back(argument::create(marg));
}
}
template<typename V>
static inline std::vector<uint>
pad_vector(command_queue &q, const V &v, uint x) {
std::vector<uint> w { v.begin(), v.end() };
w.resize(q.device().max_block_size().size(), x);
return w;
}
void
kernel::launch(command_queue &q,
const std::vector<size_t> &grid_offset,
const std::vector<size_t> &grid_size,
const std::vector<size_t> &block_size) {
const auto m = program().binary(q.device());
const auto reduced_grid_size =
map(divides(), grid_size, block_size);
void *st = exec.bind(&q, grid_offset);
// The handles are created during exec_context::bind(), so we need make
// sure to call exec_context::bind() before retrieving them.
std::vector<uint32_t *> g_handles = map([&](size_t h) {
return (uint32_t *)&exec.input[h];
}, exec.g_handles);
q.pipe->bind_compute_state(q.pipe, st);
q.pipe->bind_sampler_states(q.pipe, PIPE_SHADER_COMPUTE,
0, exec.samplers.size(),
exec.samplers.data());
q.pipe->set_sampler_views(q.pipe, PIPE_SHADER_COMPUTE, 0,
exec.sviews.size(), exec.sviews.data());
q.pipe->set_compute_resources(q.pipe, 0, exec.resources.size(),
exec.resources.data());
q.pipe->set_global_binding(q.pipe, 0, exec.g_buffers.size(),
exec.g_buffers.data(), g_handles.data());
q.pipe->launch_grid(q.pipe,
pad_vector(q, block_size, 1).data(),
pad_vector(q, reduced_grid_size, 1).data(),
find(name_equals(_name), m.syms).offset,
exec.input.data());
q.pipe->set_global_binding(q.pipe, 0, exec.g_buffers.size(), NULL, NULL);
q.pipe->set_compute_resources(q.pipe, 0, exec.resources.size(), NULL);
q.pipe->set_sampler_views(q.pipe, PIPE_SHADER_COMPUTE, 0,
exec.sviews.size(), NULL);
q.pipe->bind_sampler_states(q.pipe, PIPE_SHADER_COMPUTE, 0,
exec.samplers.size(), NULL);
exec.unbind();
}
size_t
kernel::mem_local() const {
size_t sz = 0;
for (auto &arg : args()) {
if (dynamic_cast<local_argument *>(&arg))
sz += arg.storage();
}
return sz;
}
size_t
kernel::mem_private() const {
return 0;
}
const std::string &
kernel::name() const {
return _name;
}
std::vector<size_t>
kernel::optimal_block_size(const command_queue &q,
const std::vector<size_t> &grid_size) const {
return factor::find_grid_optimal_factor<size_t>(
q.device().max_threads_per_block(), q.device().max_block_size(),
grid_size);
}
std::vector<size_t>
kernel::required_block_size() const {
return { 0, 0, 0 };
}
kernel::argument_range
kernel::args() {
return map(derefs(), _args);
}
kernel::const_argument_range
kernel::args() const {
return map(derefs(), _args);
}
const module &
kernel::module(const command_queue &q) const {
return program().binary(q.device());
}
kernel::exec_context::exec_context(kernel &kern) :
kern(kern), q(NULL), mem_local(0), st(NULL), cs() {
}
kernel::exec_context::~exec_context() {
if (st)
q->pipe->delete_compute_state(q->pipe, st);
}
void *
kernel::exec_context::bind(intrusive_ptr<command_queue> _q,
const std::vector<size_t> &grid_offset) {
std::swap(q, _q);
// Bind kernel arguments.
auto &m = kern.program().binary(q->device());
auto margs = find(name_equals(kern.name()), m.syms).args;
auto msec = find(type_equals(module::section::text), m.secs);
auto explicit_arg = kern._args.begin();
for (auto &marg : margs) {
switch (marg.semantic) {
case module::argument::general:
(*(explicit_arg++))->bind(*this, marg);
break;
case module::argument::grid_dimension: {
const cl_uint dimension = grid_offset.size();
auto arg = argument::create(marg);
arg->set(sizeof(dimension), &dimension);
arg->bind(*this, marg);
break;
}
case module::argument::grid_offset: {
for (cl_uint x : pad_vector(*q, grid_offset, 1)) {
auto arg = argument::create(marg);
arg->set(sizeof(x), &x);
arg->bind(*this, marg);
}
break;
}
}
}
// Create a new compute state if anything changed.
if (!st || q != _q ||
cs.req_local_mem != mem_local ||
cs.req_input_mem != input.size()) {
if (st)
_q->pipe->delete_compute_state(_q->pipe, st);
cs.prog = &(msec.data[0]);
cs.req_local_mem = mem_local;
cs.req_input_mem = input.size();
st = q->pipe->create_compute_state(q->pipe, &cs);
}
return st;
}
void
kernel::exec_context::unbind() {
for (auto &arg : kern.args())
arg.unbind(*this);
input.clear();
samplers.clear();
sviews.clear();
resources.clear();
g_buffers.clear();
g_handles.clear();
mem_local = 0;
}
namespace {
template<typename T>
std::vector<uint8_t>
bytes(const T& x) {
return { (uint8_t *)&x, (uint8_t *)&x + sizeof(x) };
}
///
/// Transform buffer \a v from the native byte order into the byte
/// order specified by \a e.
///
template<typename T>
void
byteswap(T &v, pipe_endian e) {
if (PIPE_ENDIAN_NATIVE != e)
std::reverse(v.begin(), v.end());
}
///
/// Pad buffer \a v to the next multiple of \a n.
///
template<typename T>
void
align(T &v, size_t n) {
v.resize(util_align_npot(v.size(), n));
}
bool
msb(const std::vector<uint8_t> &s) {
if (PIPE_ENDIAN_NATIVE == PIPE_ENDIAN_LITTLE)
return s.back() & 0x80;
else
return s.front() & 0x80;
}
///
/// Resize buffer \a v to size \a n using sign or zero extension
/// according to \a ext.
///
template<typename T>
void
extend(T &v, enum module::argument::ext_type ext, size_t n) {
const size_t m = std::min(v.size(), n);
const bool sign_ext = (ext == module::argument::sign_ext);
const uint8_t fill = (sign_ext && msb(v) ? ~0 : 0);
T w(n, fill);
if (PIPE_ENDIAN_NATIVE == PIPE_ENDIAN_LITTLE)
std::copy_n(v.begin(), m, w.begin());
else
std::copy_n(v.end() - m, m, w.end() - m);
std::swap(v, w);
}
///
/// Append buffer \a w to \a v.
///
template<typename T>
void
insert(T &v, const T &w) {
v.insert(v.end(), w.begin(), w.end());
}
///
/// Append \a n elements to the end of buffer \a v.
///
template<typename T>
size_t
allocate(T &v, size_t n) {
size_t pos = v.size();
v.resize(pos + n);
return pos;
}
}
std::unique_ptr<kernel::argument>
kernel::argument::create(const module::argument &marg) {
switch (marg.type) {
case module::argument::scalar:
return std::unique_ptr<kernel::argument>(new scalar_argument(marg.size));
case module::argument::global:
return std::unique_ptr<kernel::argument>(new global_argument);
case module::argument::local:
return std::unique_ptr<kernel::argument>(new local_argument);
case module::argument::constant:
return std::unique_ptr<kernel::argument>(new constant_argument);
case module::argument::image2d_rd:
case module::argument::image3d_rd:
return std::unique_ptr<kernel::argument>(new image_rd_argument);
case module::argument::image2d_wr:
case module::argument::image3d_wr:
return std::unique_ptr<kernel::argument>(new image_wr_argument);
case module::argument::sampler:
return std::unique_ptr<kernel::argument>(new sampler_argument);
}
throw error(CL_INVALID_KERNEL_DEFINITION);
}
kernel::argument::argument() : _set(false) {
}
bool
kernel::argument::set() const {
return _set;
}
size_t
kernel::argument::storage() const {
return 0;
}
kernel::scalar_argument::scalar_argument(size_t size) : size(size) {
}
void
kernel::scalar_argument::set(size_t size, const void *value) {
if (size != this->size)
throw error(CL_INVALID_ARG_SIZE);
v = { (uint8_t *)value, (uint8_t *)value + size };
_set = true;
}
void
kernel::scalar_argument::bind(exec_context &ctx,
const module::argument &marg) {
auto w = v;
extend(w, marg.ext_type, marg.target_size);
byteswap(w, ctx.q->device().endianness());
align(ctx.input, marg.target_align);
insert(ctx.input, w);
}
void
kernel::scalar_argument::unbind(exec_context &ctx) {
}
void
kernel::global_argument::set(size_t size, const void *value) {
if (size != sizeof(cl_mem))
throw error(CL_INVALID_ARG_SIZE);
buf = pobj<buffer>(value ? *(cl_mem *)value : NULL);
_set = true;
}
void
kernel::global_argument::bind(exec_context &ctx,
const module::argument &marg) {
align(ctx.input, marg.target_align);
if (buf) {
const resource &r = buf->resource(*ctx.q);
ctx.g_handles.push_back(ctx.input.size());
ctx.g_buffers.push_back(r.pipe);
// How to handle multi-demensional offsets?
// We don't need to. Buffer offsets are always
// one-dimensional.
auto v = bytes(r.offset[0]);
extend(v, marg.ext_type, marg.target_size);
byteswap(v, ctx.q->device().endianness());
insert(ctx.input, v);
} else {
// Null pointer.
allocate(ctx.input, marg.target_size);
}
}
void
kernel::global_argument::unbind(exec_context &ctx) {
}
size_t
kernel::local_argument::storage() const {
return _storage;
}
void
kernel::local_argument::set(size_t size, const void *value) {
if (value)
throw error(CL_INVALID_ARG_VALUE);
_storage = size;
_set = true;
}
void
kernel::local_argument::bind(exec_context &ctx,
const module::argument &marg) {
auto v = bytes(ctx.mem_local);
extend(v, module::argument::zero_ext, marg.target_size);
byteswap(v, ctx.q->device().endianness());
align(ctx.input, marg.target_align);
insert(ctx.input, v);
ctx.mem_local += _storage;
}
void
kernel::local_argument::unbind(exec_context &ctx) {
}
void
kernel::constant_argument::set(size_t size, const void *value) {
if (size != sizeof(cl_mem))
throw error(CL_INVALID_ARG_SIZE);
buf = pobj<buffer>(value ? *(cl_mem *)value : NULL);
_set = true;
}
void
kernel::constant_argument::bind(exec_context &ctx,
const module::argument &marg) {
align(ctx.input, marg.target_align);
if (buf) {
resource &r = buf->resource(*ctx.q);
auto v = bytes(ctx.resources.size() << 24 | r.offset[0]);
extend(v, module::argument::zero_ext, marg.target_size);
byteswap(v, ctx.q->device().endianness());
insert(ctx.input, v);
st = r.bind_surface(*ctx.q, false);
ctx.resources.push_back(st);
} else {
// Null pointer.
allocate(ctx.input, marg.target_size);
}
}
void
kernel::constant_argument::unbind(exec_context &ctx) {
if (buf)
buf->resource(*ctx.q).unbind_surface(*ctx.q, st);
}
void
kernel::image_rd_argument::set(size_t size, const void *value) {
if (size != sizeof(cl_mem))
throw error(CL_INVALID_ARG_SIZE);
img = &obj<image>(*(cl_mem *)value);
_set = true;
}
void
kernel::image_rd_argument::bind(exec_context &ctx,
const module::argument &marg) {
auto v = bytes(ctx.sviews.size());
extend(v, module::argument::zero_ext, marg.target_size);
byteswap(v, ctx.q->device().endianness());
align(ctx.input, marg.target_align);
insert(ctx.input, v);
st = img->resource(*ctx.q).bind_sampler_view(*ctx.q);
ctx.sviews.push_back(st);
}
void
kernel::image_rd_argument::unbind(exec_context &ctx) {
img->resource(*ctx.q).unbind_sampler_view(*ctx.q, st);
}
void
kernel::image_wr_argument::set(size_t size, const void *value) {
if (size != sizeof(cl_mem))
throw error(CL_INVALID_ARG_SIZE);
img = &obj<image>(*(cl_mem *)value);
_set = true;
}
void
kernel::image_wr_argument::bind(exec_context &ctx,
const module::argument &marg) {
auto v = bytes(ctx.resources.size());
extend(v, module::argument::zero_ext, marg.target_size);
byteswap(v, ctx.q->device().endianness());
align(ctx.input, marg.target_align);
insert(ctx.input, v);
st = img->resource(*ctx.q).bind_surface(*ctx.q, true);
ctx.resources.push_back(st);
}
void
kernel::image_wr_argument::unbind(exec_context &ctx) {
img->resource(*ctx.q).unbind_surface(*ctx.q, st);
}
void
kernel::sampler_argument::set(size_t size, const void *value) {
if (size != sizeof(cl_sampler))
throw error(CL_INVALID_ARG_SIZE);
s = &obj(*(cl_sampler *)value);
_set = true;
}
void
kernel::sampler_argument::bind(exec_context &ctx,
const module::argument &marg) {
st = s->bind(*ctx.q);
ctx.samplers.push_back(st);
}
void
kernel::sampler_argument::unbind(exec_context &ctx) {
s->unbind(*ctx.q, st);
}