919 lines
35 KiB
C++
919 lines
35 KiB
C++
// Copyright 2009-2021 Intel Corporation
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
#include "bvh_intersector_hybrid.h"
|
|
#include "bvh_traverser1.h"
|
|
#include "node_intersector1.h"
|
|
#include "node_intersector_packet.h"
|
|
|
|
#include "../geometry/intersector_iterators.h"
|
|
#include "../geometry/triangle_intersector.h"
|
|
#include "../geometry/trianglev_intersector.h"
|
|
#include "../geometry/trianglev_mb_intersector.h"
|
|
#include "../geometry/trianglei_intersector.h"
|
|
#include "../geometry/quadv_intersector.h"
|
|
#include "../geometry/quadi_intersector.h"
|
|
#include "../geometry/curveNv_intersector.h"
|
|
#include "../geometry/curveNi_intersector.h"
|
|
#include "../geometry/curveNi_mb_intersector.h"
|
|
#include "../geometry/linei_intersector.h"
|
|
#include "../geometry/subdivpatch1_intersector.h"
|
|
#include "../geometry/object_intersector.h"
|
|
#include "../geometry/instance_intersector.h"
|
|
#include "../geometry/instance_array_intersector.h"
|
|
#include "../geometry/subgrid_intersector.h"
|
|
#include "../geometry/subgrid_mb_intersector.h"
|
|
#include "../geometry/curve_intersector_virtual.h"
|
|
|
|
#define SWITCH_DURING_DOWN_TRAVERSAL 1
|
|
#define FORCE_SINGLE_MODE 0
|
|
|
|
#define ENABLE_FAST_COHERENT_CODEPATHS 1
|
|
|
|
namespace embree
|
|
{
|
|
namespace isa
|
|
{
|
|
template<int N, int K, int types, bool robust, typename PrimitiveIntersectorK, bool single>
|
|
void BVHNIntersectorKHybrid<N, K, types, robust, PrimitiveIntersectorK, single>::intersect1(Accel::Intersectors* This,
|
|
const BVH* bvh,
|
|
NodeRef root,
|
|
size_t k,
|
|
Precalculations& pre,
|
|
RayHitK<K>& ray,
|
|
const TravRayK<K, robust>& tray,
|
|
RayQueryContext* context)
|
|
{
|
|
/* stack state */
|
|
StackItemT<NodeRef> stack[stackSizeSingle]; // stack of nodes
|
|
StackItemT<NodeRef>* stackPtr = stack + 1; // current stack pointer
|
|
StackItemT<NodeRef>* stackEnd = stack + stackSizeSingle;
|
|
stack[0].ptr = root;
|
|
stack[0].dist = neg_inf;
|
|
|
|
/* load the ray into SIMD registers */
|
|
TravRay<N,robust> tray1;
|
|
tray1.template init<K>(k, tray.org, tray.dir, tray.rdir, tray.nearXYZ, tray.tnear[k], tray.tfar[k]);
|
|
|
|
/* pop loop */
|
|
while (true) pop:
|
|
{
|
|
/* pop next node */
|
|
if (unlikely(stackPtr == stack)) break;
|
|
stackPtr--;
|
|
NodeRef cur = NodeRef(stackPtr->ptr);
|
|
|
|
/* if popped node is too far, pop next one */
|
|
if (unlikely(*(float*)&stackPtr->dist > ray.tfar[k]))
|
|
continue;
|
|
|
|
/* downtraversal loop */
|
|
while (true)
|
|
{
|
|
/* intersect node */
|
|
size_t mask; vfloat<N> tNear;
|
|
STAT3(normal.trav_nodes, 1, 1, 1);
|
|
bool nodeIntersected = BVHNNodeIntersector1<N, types, robust>::intersect(cur, tray1, ray.time()[k], tNear, mask);
|
|
if (unlikely(!nodeIntersected)) { STAT3(normal.trav_nodes,-1,-1,-1); break; }
|
|
|
|
/* if no child is hit, pop next node */
|
|
if (unlikely(mask == 0))
|
|
goto pop;
|
|
|
|
/* select next child and push other children */
|
|
BVHNNodeTraverser1Hit<N, types>::traverseClosestHit(cur, mask, tNear, stackPtr, stackEnd);
|
|
}
|
|
|
|
/* this is a leaf node */
|
|
assert(cur != BVH::emptyNode);
|
|
STAT3(normal.trav_leaves, 1, 1, 1);
|
|
size_t num; Primitive* prim = (Primitive*)cur.leaf(num);
|
|
|
|
size_t lazy_node = 0;
|
|
PrimitiveIntersectorK::intersect(This, pre, ray, k, context, prim, num, tray1, lazy_node);
|
|
|
|
tray1.tfar = ray.tfar[k];
|
|
|
|
if (unlikely(lazy_node)) {
|
|
stackPtr->ptr = lazy_node;
|
|
stackPtr->dist = neg_inf;
|
|
stackPtr++;
|
|
}
|
|
}
|
|
}
|
|
|
|
template<int N, int K, int types, bool robust, typename PrimitiveIntersectorK, bool single>
|
|
void BVHNIntersectorKHybrid<N, K, types, robust, PrimitiveIntersectorK, single>::intersect(vint<K>* __restrict__ valid_i,
|
|
Accel::Intersectors* __restrict__ This,
|
|
RayHitK<K>& __restrict__ ray,
|
|
RayQueryContext* __restrict__ context)
|
|
{
|
|
BVH* __restrict__ bvh = (BVH*)This->ptr;
|
|
|
|
/* we may traverse an empty BVH in case all geometry was invalid */
|
|
if (bvh->root == BVH::emptyNode)
|
|
return;
|
|
|
|
#if ENABLE_FAST_COHERENT_CODEPATHS == 1
|
|
assert(context);
|
|
if (unlikely(types == BVH_AN1 && context->user && context->isCoherent()))
|
|
{
|
|
intersectCoherent(valid_i, This, ray, context);
|
|
return;
|
|
}
|
|
#endif
|
|
|
|
/* filter out invalid rays */
|
|
vbool<K> valid = *valid_i == -1;
|
|
#if defined(EMBREE_IGNORE_INVALID_RAYS)
|
|
valid &= ray.valid();
|
|
#endif
|
|
|
|
/* return if there are no valid rays */
|
|
size_t valid_bits = movemask(valid);
|
|
|
|
#if defined(__AVX__)
|
|
STAT3(normal.trav_hit_boxes[popcnt(movemask(valid))], 1, 1, 1);
|
|
#endif
|
|
|
|
if (unlikely(valid_bits == 0)) return;
|
|
|
|
/* verify correct input */
|
|
assert(all(valid, ray.valid()));
|
|
assert(all(valid, ray.tnear() >= 0.0f));
|
|
assert(!(types & BVH_MB) || all(valid, (ray.time() >= 0.0f) & (ray.time() <= 1.0f)));
|
|
Precalculations pre(valid, ray);
|
|
|
|
/* load ray */
|
|
TravRayK<K, robust> tray(ray.org, ray.dir, single ? N : 0);
|
|
const vfloat<K> org_ray_tnear = max(ray.tnear(), 0.0f);
|
|
const vfloat<K> org_ray_tfar = max(ray.tfar , 0.0f);
|
|
|
|
if (single)
|
|
{
|
|
tray.tnear = select(valid, org_ray_tnear, vfloat<K>(pos_inf));
|
|
tray.tfar = select(valid, org_ray_tfar , vfloat<K>(neg_inf));
|
|
|
|
for (; valid_bits!=0; ) {
|
|
const size_t i = bscf(valid_bits);
|
|
intersect1(This, bvh, bvh->root, i, pre, ray, tray, context);
|
|
}
|
|
return;
|
|
}
|
|
|
|
/* determine switch threshold based on flags */
|
|
const size_t switchThreshold = (context->user && context->isCoherent()) ? 2 : switchThresholdIncoherent;
|
|
|
|
vint<K> octant = ray.octant();
|
|
octant = select(valid, octant, vint<K>(0xffffffff));
|
|
|
|
/* test whether we have ray with opposing direction signs in the packet */
|
|
bool split = false;
|
|
{
|
|
size_t bits = valid_bits;
|
|
vbool<K> vsplit( false );
|
|
do
|
|
{
|
|
const size_t valid_index = bsf(bits);
|
|
vbool<K> octant_valid = octant[valid_index] == octant;
|
|
bits &= ~(size_t)movemask(octant_valid);
|
|
vsplit |= vint<K>(octant[valid_index]) == (octant^vint<K>(0x7));
|
|
} while (bits);
|
|
if (any(vsplit)) split = true;
|
|
}
|
|
|
|
do
|
|
{
|
|
const size_t valid_index = bsf(valid_bits);
|
|
const vint<K> diff_octant = vint<K>(octant[valid_index])^octant;
|
|
const vint<K> count_diff_octant = \
|
|
((diff_octant >> 2) & 1) +
|
|
((diff_octant >> 1) & 1) +
|
|
((diff_octant >> 0) & 1);
|
|
|
|
vbool<K> octant_valid = (count_diff_octant <= 1) & (octant != vint<K>(0xffffffff));
|
|
if (!single || !split) octant_valid = valid; // deactivate octant sorting in pure chunk mode, otherwise instance traversal performance goes down
|
|
|
|
|
|
octant = select(octant_valid,vint<K>(0xffffffff),octant);
|
|
valid_bits &= ~(size_t)movemask(octant_valid);
|
|
|
|
tray.tnear = select(octant_valid, org_ray_tnear, vfloat<K>(pos_inf));
|
|
tray.tfar = select(octant_valid, org_ray_tfar , vfloat<K>(neg_inf));
|
|
|
|
/* allocate stack and push root node */
|
|
vfloat<K> stack_near[stackSizeChunk];
|
|
NodeRef stack_node[stackSizeChunk];
|
|
stack_node[0] = BVH::invalidNode;
|
|
stack_near[0] = inf;
|
|
stack_node[1] = bvh->root;
|
|
stack_near[1] = tray.tnear;
|
|
NodeRef* stackEnd MAYBE_UNUSED = stack_node+stackSizeChunk;
|
|
NodeRef* __restrict__ sptr_node = stack_node + 2;
|
|
vfloat<K>* __restrict__ sptr_near = stack_near + 2;
|
|
|
|
while (1) pop:
|
|
{
|
|
/* pop next node from stack */
|
|
assert(sptr_node > stack_node);
|
|
sptr_node--;
|
|
sptr_near--;
|
|
NodeRef cur = *sptr_node;
|
|
if (unlikely(cur == BVH::invalidNode)) {
|
|
assert(sptr_node == stack_node);
|
|
break;
|
|
}
|
|
|
|
/* cull node if behind closest hit point */
|
|
vfloat<K> curDist = *sptr_near;
|
|
const vbool<K> active = curDist < tray.tfar;
|
|
if (unlikely(none(active)))
|
|
continue;
|
|
|
|
/* switch to single ray traversal */
|
|
#if (!defined(__WIN32__) || defined(__X86_64__)) && ((defined(__aarch64__)) || defined(__SSE4_2__))
|
|
#if FORCE_SINGLE_MODE == 0
|
|
if (single)
|
|
#endif
|
|
{
|
|
size_t bits = movemask(active);
|
|
#if FORCE_SINGLE_MODE == 0
|
|
if (unlikely(popcnt(bits) <= switchThreshold))
|
|
#endif
|
|
{
|
|
for (; bits!=0; ) {
|
|
const size_t i = bscf(bits);
|
|
intersect1(This, bvh, cur, i, pre, ray, tray, context);
|
|
}
|
|
tray.tfar = min(tray.tfar, ray.tfar);
|
|
continue;
|
|
}
|
|
}
|
|
#endif
|
|
while (likely(!cur.isLeaf()))
|
|
{
|
|
/* process nodes */
|
|
const vbool<K> valid_node = tray.tfar > curDist;
|
|
STAT3(normal.trav_nodes, 1, popcnt(valid_node), K);
|
|
const NodeRef nodeRef = cur;
|
|
const BaseNode* __restrict__ const node = nodeRef.baseNode();
|
|
|
|
/* set cur to invalid */
|
|
cur = BVH::emptyNode;
|
|
curDist = pos_inf;
|
|
|
|
size_t num_child_hits = 0;
|
|
|
|
for (unsigned i = 0; i < N; i++)
|
|
{
|
|
const NodeRef child = node->children[i];
|
|
if (unlikely(child == BVH::emptyNode)) break;
|
|
vfloat<K> lnearP;
|
|
vbool<K> lhit = valid_node;
|
|
BVHNNodeIntersectorK<N, K, types, robust>::intersect(nodeRef, i, tray, ray.time(), lnearP, lhit);
|
|
|
|
/* if we hit the child we choose to continue with that child if it
|
|
is closer than the current next child, or we push it onto the stack */
|
|
if (likely(any(lhit)))
|
|
{
|
|
assert(sptr_node < stackEnd);
|
|
assert(child != BVH::emptyNode);
|
|
const vfloat<K> childDist = select(lhit, lnearP, inf);
|
|
/* push cur node onto stack and continue with hit child */
|
|
if (any(childDist < curDist))
|
|
{
|
|
if (likely(cur != BVH::emptyNode)) {
|
|
num_child_hits++;
|
|
*sptr_node = cur; sptr_node++;
|
|
*sptr_near = curDist; sptr_near++;
|
|
}
|
|
curDist = childDist;
|
|
cur = child;
|
|
}
|
|
|
|
/* push hit child onto stack */
|
|
else {
|
|
num_child_hits++;
|
|
*sptr_node = child; sptr_node++;
|
|
*sptr_near = childDist; sptr_near++;
|
|
}
|
|
}
|
|
}
|
|
|
|
#if defined(__AVX__)
|
|
//STAT3(normal.trav_hit_boxes[num_child_hits], 1, 1, 1);
|
|
#endif
|
|
|
|
if (unlikely(cur == BVH::emptyNode))
|
|
goto pop;
|
|
|
|
/* improved distance sorting for 3 or more hits */
|
|
if (unlikely(num_child_hits >= 2))
|
|
{
|
|
if (any(sptr_near[-2] < sptr_near[-1]))
|
|
{
|
|
std::swap(sptr_near[-2],sptr_near[-1]);
|
|
std::swap(sptr_node[-2],sptr_node[-1]);
|
|
}
|
|
if (unlikely(num_child_hits >= 3))
|
|
{
|
|
if (any(sptr_near[-3] < sptr_near[-1]))
|
|
{
|
|
std::swap(sptr_near[-3],sptr_near[-1]);
|
|
std::swap(sptr_node[-3],sptr_node[-1]);
|
|
}
|
|
if (any(sptr_near[-3] < sptr_near[-2]))
|
|
{
|
|
std::swap(sptr_near[-3],sptr_near[-2]);
|
|
std::swap(sptr_node[-3],sptr_node[-2]);
|
|
}
|
|
}
|
|
}
|
|
|
|
#if SWITCH_DURING_DOWN_TRAVERSAL == 1
|
|
if (single)
|
|
{
|
|
// seems to be the best place for testing utilization
|
|
if (unlikely(popcnt(tray.tfar > curDist) <= switchThreshold))
|
|
{
|
|
*sptr_node++ = cur;
|
|
*sptr_near++ = curDist;
|
|
goto pop;
|
|
}
|
|
}
|
|
#endif
|
|
}
|
|
|
|
/* return if stack is empty */
|
|
if (unlikely(cur == BVH::invalidNode)) {
|
|
assert(sptr_node == stack_node);
|
|
break;
|
|
}
|
|
|
|
/* intersect leaf */
|
|
assert(cur != BVH::emptyNode);
|
|
const vbool<K> valid_leaf = tray.tfar > curDist;
|
|
STAT3(normal.trav_leaves, 1, popcnt(valid_leaf), K);
|
|
if (unlikely(none(valid_leaf))) continue;
|
|
size_t items; const Primitive* prim = (Primitive*)cur.leaf(items);
|
|
|
|
size_t lazy_node = 0;
|
|
PrimitiveIntersectorK::intersect(valid_leaf, This, pre, ray, context, prim, items, tray, lazy_node);
|
|
tray.tfar = select(valid_leaf, ray.tfar, tray.tfar);
|
|
|
|
if (unlikely(lazy_node)) {
|
|
*sptr_node = lazy_node; sptr_node++;
|
|
*sptr_near = neg_inf; sptr_near++;
|
|
}
|
|
}
|
|
} while(valid_bits);
|
|
}
|
|
|
|
|
|
template<int N, int K, int types, bool robust, typename PrimitiveIntersectorK, bool single>
|
|
void BVHNIntersectorKHybrid<N, K, types, robust, PrimitiveIntersectorK, single>::intersectCoherent(vint<K>* __restrict__ valid_i,
|
|
Accel::Intersectors* __restrict__ This,
|
|
RayHitK<K>& __restrict__ ray,
|
|
RayQueryContext* context)
|
|
{
|
|
BVH* __restrict__ bvh = (BVH*)This->ptr;
|
|
|
|
/* filter out invalid rays */
|
|
vbool<K> valid = *valid_i == -1;
|
|
#if defined(EMBREE_IGNORE_INVALID_RAYS)
|
|
valid &= ray.valid();
|
|
#endif
|
|
|
|
/* return if there are no valid rays */
|
|
size_t valid_bits = movemask(valid);
|
|
if (unlikely(valid_bits == 0)) return;
|
|
|
|
/* verify correct input */
|
|
assert(all(valid, ray.valid()));
|
|
assert(all(valid, ray.tnear() >= 0.0f));
|
|
assert(!(types & BVH_MB) || all(valid, (ray.time() >= 0.0f) & (ray.time() <= 1.0f)));
|
|
Precalculations pre(valid, ray);
|
|
|
|
/* load ray */
|
|
TravRayK<K, robust> tray(ray.org, ray.dir, single ? N : 0);
|
|
const vfloat<K> org_ray_tnear = max(ray.tnear(), 0.0f);
|
|
const vfloat<K> org_ray_tfar = max(ray.tfar , 0.0f);
|
|
|
|
vint<K> octant = ray.octant();
|
|
octant = select(valid, octant, vint<K>(0xffffffff));
|
|
|
|
do
|
|
{
|
|
const size_t valid_index = bsf(valid_bits);
|
|
const vbool<K> octant_valid = octant[valid_index] == octant;
|
|
valid_bits &= ~(size_t)movemask(octant_valid);
|
|
|
|
tray.tnear = select(octant_valid, org_ray_tnear, vfloat<K>(pos_inf));
|
|
tray.tfar = select(octant_valid, org_ray_tfar , vfloat<K>(neg_inf));
|
|
|
|
Frustum<robust> frustum;
|
|
frustum.template init<K>(octant_valid, tray.org, tray.rdir, tray.tnear, tray.tfar, N);
|
|
|
|
StackItemT<NodeRef> stack[stackSizeSingle]; // stack of nodes
|
|
StackItemT<NodeRef>* stackPtr = stack + 1; // current stack pointer
|
|
stack[0].ptr = bvh->root;
|
|
stack[0].dist = neg_inf;
|
|
|
|
while (1) pop:
|
|
{
|
|
/* pop next node from stack */
|
|
if (unlikely(stackPtr == stack)) break;
|
|
|
|
stackPtr--;
|
|
NodeRef cur = NodeRef(stackPtr->ptr);
|
|
|
|
/* cull node if behind closest hit point */
|
|
vfloat<K> curDist = *(float*)&stackPtr->dist;
|
|
const vbool<K> active = curDist < tray.tfar;
|
|
if (unlikely(none(active))) continue;
|
|
|
|
while (likely(!cur.isLeaf()))
|
|
{
|
|
/* process nodes */
|
|
//STAT3(normal.trav_nodes, 1, popcnt(valid_node), K);
|
|
const NodeRef nodeRef = cur;
|
|
const AABBNode* __restrict__ const node = nodeRef.getAABBNode();
|
|
|
|
vfloat<N> fmin;
|
|
size_t m_frustum_node = intersectNodeFrustum<N>(node, frustum, fmin);
|
|
|
|
if (unlikely(!m_frustum_node)) goto pop;
|
|
cur = BVH::emptyNode;
|
|
curDist = pos_inf;
|
|
|
|
#if defined(__AVX__)
|
|
//STAT3(normal.trav_hit_boxes[popcnt(m_frustum_node)], 1, 1, 1);
|
|
#endif
|
|
size_t num_child_hits = 0;
|
|
do {
|
|
const size_t i = bscf(m_frustum_node);
|
|
vfloat<K> lnearP;
|
|
vbool<K> lhit = false; // motion blur is not supported, so the initial value will be ignored
|
|
STAT3(normal.trav_nodes, 1, 1, 1);
|
|
BVHNNodeIntersectorK<N, K, types, robust>::intersect(nodeRef, i, tray, ray.time(), lnearP, lhit);
|
|
|
|
if (likely(any(lhit)))
|
|
{
|
|
const vfloat<K> childDist = fmin[i];
|
|
const NodeRef child = node->child(i);
|
|
BVHN<N>::prefetch(child);
|
|
if (any(childDist < curDist))
|
|
{
|
|
if (likely(cur != BVH::emptyNode)) {
|
|
num_child_hits++;
|
|
stackPtr->ptr = cur;
|
|
*(float*)&stackPtr->dist = toScalar(curDist);
|
|
stackPtr++;
|
|
}
|
|
curDist = childDist;
|
|
cur = child;
|
|
}
|
|
/* push hit child onto stack */
|
|
else {
|
|
num_child_hits++;
|
|
stackPtr->ptr = child;
|
|
*(float*)&stackPtr->dist = toScalar(childDist);
|
|
stackPtr++;
|
|
}
|
|
}
|
|
} while(m_frustum_node);
|
|
|
|
if (unlikely(cur == BVH::emptyNode)) goto pop;
|
|
|
|
/* improved distance sorting for 3 or more hits */
|
|
if (unlikely(num_child_hits >= 2))
|
|
{
|
|
if (stackPtr[-2].dist < stackPtr[-1].dist)
|
|
std::swap(stackPtr[-2],stackPtr[-1]);
|
|
if (unlikely(num_child_hits >= 3))
|
|
{
|
|
if (stackPtr[-3].dist < stackPtr[-1].dist)
|
|
std::swap(stackPtr[-3],stackPtr[-1]);
|
|
if (stackPtr[-3].dist < stackPtr[-2].dist)
|
|
std::swap(stackPtr[-3],stackPtr[-2]);
|
|
}
|
|
}
|
|
}
|
|
|
|
/* intersect leaf */
|
|
assert(cur != BVH::invalidNode);
|
|
assert(cur != BVH::emptyNode);
|
|
const vbool<K> valid_leaf = tray.tfar > curDist;
|
|
STAT3(normal.trav_leaves, 1, popcnt(valid_leaf), K);
|
|
if (unlikely(none(valid_leaf))) continue;
|
|
size_t items; const Primitive* prim = (Primitive*)cur.leaf(items);
|
|
|
|
size_t lazy_node = 0;
|
|
PrimitiveIntersectorK::intersect(valid_leaf, This, pre, ray, context, prim, items, tray, lazy_node);
|
|
|
|
/* reduce max distance interval on successful intersection */
|
|
if (likely(any((ray.tfar < tray.tfar) & valid_leaf)))
|
|
{
|
|
tray.tfar = select(valid_leaf, ray.tfar, tray.tfar);
|
|
frustum.template updateMaxDist<K>(tray.tfar);
|
|
}
|
|
|
|
if (unlikely(lazy_node)) {
|
|
stackPtr->ptr = lazy_node;
|
|
stackPtr->dist = neg_inf;
|
|
stackPtr++;
|
|
}
|
|
}
|
|
|
|
} while(valid_bits);
|
|
}
|
|
|
|
// ===================================================================================================================================================================
|
|
// ===================================================================================================================================================================
|
|
// ===================================================================================================================================================================
|
|
|
|
template<int N, int K, int types, bool robust, typename PrimitiveIntersectorK, bool single>
|
|
bool BVHNIntersectorKHybrid<N, K, types, robust, PrimitiveIntersectorK, single>::occluded1(Accel::Intersectors* This,
|
|
const BVH* bvh,
|
|
NodeRef root,
|
|
size_t k,
|
|
Precalculations& pre,
|
|
RayK<K>& ray,
|
|
const TravRayK<K, robust>& tray,
|
|
RayQueryContext* context)
|
|
{
|
|
/* stack state */
|
|
NodeRef stack[stackSizeSingle]; // stack of nodes that still need to get traversed
|
|
NodeRef* stackPtr = stack+1; // current stack pointer
|
|
NodeRef* stackEnd = stack+stackSizeSingle;
|
|
stack[0] = root;
|
|
|
|
/* load the ray into SIMD registers */
|
|
TravRay<N,robust> tray1;
|
|
tray1.template init<K>(k, tray.org, tray.dir, tray.rdir, tray.nearXYZ, tray.tnear[k], tray.tfar[k]);
|
|
|
|
/* pop loop */
|
|
while (true) pop:
|
|
{
|
|
/* pop next node */
|
|
if (unlikely(stackPtr == stack)) break;
|
|
stackPtr--;
|
|
NodeRef cur = (NodeRef)*stackPtr;
|
|
|
|
/* downtraversal loop */
|
|
while (true)
|
|
{
|
|
/* intersect node */
|
|
size_t mask; vfloat<N> tNear;
|
|
STAT3(shadow.trav_nodes, 1, 1, 1);
|
|
bool nodeIntersected = BVHNNodeIntersector1<N, types, robust>::intersect(cur, tray1, ray.time()[k], tNear, mask);
|
|
if (unlikely(!nodeIntersected)) { STAT3(shadow.trav_nodes,-1,-1,-1); break; }
|
|
|
|
/* if no child is hit, pop next node */
|
|
if (unlikely(mask == 0))
|
|
goto pop;
|
|
|
|
/* select next child and push other children */
|
|
BVHNNodeTraverser1Hit<N, types>::traverseAnyHit(cur, mask, tNear, stackPtr, stackEnd);
|
|
}
|
|
|
|
/* this is a leaf node */
|
|
assert(cur != BVH::emptyNode);
|
|
STAT3(shadow.trav_leaves, 1, 1, 1);
|
|
size_t num; Primitive* prim = (Primitive*)cur.leaf(num);
|
|
|
|
size_t lazy_node = 0;
|
|
if (PrimitiveIntersectorK::occluded(This, pre, ray, k, context, prim, num, tray1, lazy_node)) {
|
|
ray.tfar[k] = neg_inf;
|
|
return true;
|
|
}
|
|
|
|
if (unlikely(lazy_node)) {
|
|
*stackPtr = lazy_node;
|
|
stackPtr++;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
template<int N, int K, int types, bool robust, typename PrimitiveIntersectorK, bool single>
|
|
void BVHNIntersectorKHybrid<N, K, types, robust, PrimitiveIntersectorK, single>::occluded(vint<K>* __restrict__ valid_i,
|
|
Accel::Intersectors* __restrict__ This,
|
|
RayK<K>& __restrict__ ray,
|
|
RayQueryContext* context)
|
|
{
|
|
BVH* __restrict__ bvh = (BVH*)This->ptr;
|
|
|
|
/* we may traverse an empty BVH in case all geometry was invalid */
|
|
if (bvh->root == BVH::emptyNode)
|
|
return;
|
|
|
|
#if ENABLE_FAST_COHERENT_CODEPATHS == 1
|
|
assert(context);
|
|
if (unlikely(types == BVH_AN1 && context->user && context->isCoherent()))
|
|
{
|
|
occludedCoherent(valid_i, This, ray, context);
|
|
return;
|
|
}
|
|
#endif
|
|
|
|
/* filter out already occluded and invalid rays */
|
|
vbool<K> valid = (*valid_i == -1) & (ray.tfar >= 0.0f);
|
|
#if defined(EMBREE_IGNORE_INVALID_RAYS)
|
|
valid &= ray.valid();
|
|
#endif
|
|
|
|
/* return if there are no valid rays */
|
|
const size_t valid_bits = movemask(valid);
|
|
if (unlikely(valid_bits == 0)) return;
|
|
|
|
/* verify correct input */
|
|
assert(all(valid, ray.valid()));
|
|
assert(all(valid, ray.tnear() >= 0.0f));
|
|
assert(!(types & BVH_MB) || all(valid, (ray.time() >= 0.0f) & (ray.time() <= 1.0f)));
|
|
Precalculations pre(valid, ray);
|
|
|
|
/* load ray */
|
|
TravRayK<K, robust> tray(ray.org, ray.dir, single ? N : 0);
|
|
const vfloat<K> org_ray_tnear = max(ray.tnear(), 0.0f);
|
|
const vfloat<K> org_ray_tfar = max(ray.tfar , 0.0f);
|
|
|
|
tray.tnear = select(valid, org_ray_tnear, vfloat<K>(pos_inf));
|
|
tray.tfar = select(valid, org_ray_tfar , vfloat<K>(neg_inf));
|
|
|
|
vbool<K> terminated = !valid;
|
|
const vfloat<K> inf = vfloat<K>(pos_inf);
|
|
|
|
/* determine switch threshold based on flags */
|
|
const size_t switchThreshold = (context->user && context->isCoherent()) ? 2 : switchThresholdIncoherent;
|
|
|
|
/* allocate stack and push root node */
|
|
vfloat<K> stack_near[stackSizeChunk];
|
|
NodeRef stack_node[stackSizeChunk];
|
|
stack_node[0] = BVH::invalidNode;
|
|
stack_near[0] = inf;
|
|
stack_node[1] = bvh->root;
|
|
stack_near[1] = tray.tnear;
|
|
NodeRef* stackEnd MAYBE_UNUSED = stack_node+stackSizeChunk;
|
|
NodeRef* __restrict__ sptr_node = stack_node + 2;
|
|
vfloat<K>* __restrict__ sptr_near = stack_near + 2;
|
|
|
|
while (1) pop:
|
|
{
|
|
/* pop next node from stack */
|
|
assert(sptr_node > stack_node);
|
|
sptr_node--;
|
|
sptr_near--;
|
|
NodeRef cur = *sptr_node;
|
|
if (unlikely(cur == BVH::invalidNode)) {
|
|
assert(sptr_node == stack_node);
|
|
break;
|
|
}
|
|
|
|
/* cull node if behind closest hit point */
|
|
vfloat<K> curDist = *sptr_near;
|
|
const vbool<K> active = curDist < tray.tfar;
|
|
if (unlikely(none(active)))
|
|
continue;
|
|
|
|
/* switch to single ray traversal */
|
|
#if (!defined(__WIN32__) || defined(__X86_64__)) && ((defined(__aarch64__)) || defined(__SSE4_2__))
|
|
#if FORCE_SINGLE_MODE == 0
|
|
if (single)
|
|
#endif
|
|
{
|
|
size_t bits = movemask(active);
|
|
#if FORCE_SINGLE_MODE == 0
|
|
if (unlikely(popcnt(bits) <= switchThreshold))
|
|
#endif
|
|
{
|
|
for (; bits!=0; ) {
|
|
const size_t i = bscf(bits);
|
|
if (occluded1(This, bvh, cur, i, pre, ray, tray, context))
|
|
set(terminated, i);
|
|
}
|
|
if (all(terminated)) break;
|
|
tray.tfar = select(terminated, vfloat<K>(neg_inf), tray.tfar);
|
|
continue;
|
|
}
|
|
}
|
|
#endif
|
|
|
|
while (likely(!cur.isLeaf()))
|
|
{
|
|
/* process nodes */
|
|
const vbool<K> valid_node = tray.tfar > curDist;
|
|
STAT3(shadow.trav_nodes, 1, popcnt(valid_node), K);
|
|
const NodeRef nodeRef = cur;
|
|
const BaseNode* __restrict__ const node = nodeRef.baseNode();
|
|
|
|
/* set cur to invalid */
|
|
cur = BVH::emptyNode;
|
|
curDist = pos_inf;
|
|
|
|
for (unsigned i = 0; i < N; i++)
|
|
{
|
|
const NodeRef child = node->children[i];
|
|
if (unlikely(child == BVH::emptyNode)) break;
|
|
vfloat<K> lnearP;
|
|
vbool<K> lhit = valid_node;
|
|
BVHNNodeIntersectorK<N, K, types, robust>::intersect(nodeRef, i, tray, ray.time(), lnearP, lhit);
|
|
|
|
/* if we hit the child we push the previously hit node onto the stack, and continue with the currently hit child */
|
|
if (likely(any(lhit)))
|
|
{
|
|
assert(sptr_node < stackEnd);
|
|
assert(child != BVH::emptyNode);
|
|
const vfloat<K> childDist = select(lhit, lnearP, inf);
|
|
|
|
/* push 'cur' node onto stack and continue with hit child */
|
|
if (likely(cur != BVH::emptyNode)) {
|
|
*sptr_node = cur; sptr_node++;
|
|
*sptr_near = curDist; sptr_near++;
|
|
}
|
|
curDist = childDist;
|
|
cur = child;
|
|
}
|
|
}
|
|
if (unlikely(cur == BVH::emptyNode))
|
|
goto pop;
|
|
|
|
#if SWITCH_DURING_DOWN_TRAVERSAL == 1
|
|
if (single)
|
|
{
|
|
// seems to be the best place for testing utilization
|
|
if (unlikely(popcnt(tray.tfar > curDist) <= switchThreshold))
|
|
{
|
|
*sptr_node++ = cur;
|
|
*sptr_near++ = curDist;
|
|
goto pop;
|
|
}
|
|
}
|
|
#endif
|
|
}
|
|
|
|
/* return if stack is empty */
|
|
if (unlikely(cur == BVH::invalidNode)) {
|
|
assert(sptr_node == stack_node);
|
|
break;
|
|
}
|
|
|
|
|
|
/* intersect leaf */
|
|
assert(cur != BVH::emptyNode);
|
|
const vbool<K> valid_leaf = tray.tfar > curDist;
|
|
STAT3(shadow.trav_leaves, 1, popcnt(valid_leaf), K);
|
|
if (unlikely(none(valid_leaf))) continue;
|
|
size_t items; const Primitive* prim = (Primitive*) cur.leaf(items);
|
|
|
|
size_t lazy_node = 0;
|
|
terminated |= PrimitiveIntersectorK::occluded(!terminated, This, pre, ray, context, prim, items, tray, lazy_node);
|
|
if (all(terminated)) break;
|
|
tray.tfar = select(terminated, vfloat<K>(neg_inf), tray.tfar); // ignore node intersections for terminated rays
|
|
|
|
if (unlikely(lazy_node)) {
|
|
*sptr_node = lazy_node; sptr_node++;
|
|
*sptr_near = neg_inf; sptr_near++;
|
|
}
|
|
}
|
|
|
|
vfloat<K>::store(valid & terminated, &ray.tfar, neg_inf);
|
|
}
|
|
|
|
|
|
template<int N, int K, int types, bool robust, typename PrimitiveIntersectorK, bool single>
|
|
void BVHNIntersectorKHybrid<N, K, types, robust, PrimitiveIntersectorK, single>::occludedCoherent(vint<K>* __restrict__ valid_i,
|
|
Accel::Intersectors* __restrict__ This,
|
|
RayK<K>& __restrict__ ray,
|
|
RayQueryContext* context)
|
|
{
|
|
BVH* __restrict__ bvh = (BVH*)This->ptr;
|
|
|
|
/* filter out invalid rays */
|
|
vbool<K> valid = *valid_i == -1;
|
|
#if defined(EMBREE_IGNORE_INVALID_RAYS)
|
|
valid &= ray.valid();
|
|
#endif
|
|
|
|
/* return if there are no valid rays */
|
|
size_t valid_bits = movemask(valid);
|
|
if (unlikely(valid_bits == 0)) return;
|
|
|
|
/* verify correct input */
|
|
assert(all(valid, ray.valid()));
|
|
assert(all(valid, ray.tnear() >= 0.0f));
|
|
assert(!(types & BVH_MB) || all(valid, (ray.time() >= 0.0f) & (ray.time() <= 1.0f)));
|
|
Precalculations pre(valid,ray);
|
|
|
|
/* load ray */
|
|
TravRayK<K, robust> tray(ray.org, ray.dir, single ? N : 0);
|
|
const vfloat<K> org_ray_tnear = max(ray.tnear(), 0.0f);
|
|
const vfloat<K> org_ray_tfar = max(ray.tfar , 0.0f);
|
|
|
|
vbool<K> terminated = !valid;
|
|
|
|
vint<K> octant = ray.octant();
|
|
octant = select(valid, octant, vint<K>(0xffffffff));
|
|
|
|
do
|
|
{
|
|
const size_t valid_index = bsf(valid_bits);
|
|
vbool<K> octant_valid = octant[valid_index] == octant;
|
|
valid_bits &= ~(size_t)movemask(octant_valid);
|
|
|
|
tray.tnear = select(octant_valid, org_ray_tnear, vfloat<K>(pos_inf));
|
|
tray.tfar = select(octant_valid, org_ray_tfar, vfloat<K>(neg_inf));
|
|
|
|
Frustum<robust> frustum;
|
|
frustum.template init<K>(octant_valid, tray.org, tray.rdir, tray.tnear, tray.tfar, N);
|
|
|
|
StackItemMaskT<NodeRef> stack[stackSizeSingle]; // stack of nodes
|
|
StackItemMaskT<NodeRef>* stackPtr = stack + 1; // current stack pointer
|
|
stack[0].ptr = bvh->root;
|
|
stack[0].mask = movemask(octant_valid);
|
|
|
|
while (1) pop:
|
|
{
|
|
/* pop next node from stack */
|
|
if (unlikely(stackPtr == stack)) break;
|
|
|
|
stackPtr--;
|
|
NodeRef cur = NodeRef(stackPtr->ptr);
|
|
|
|
/* cull node of active rays have already been terminated */
|
|
size_t m_active = (size_t)stackPtr->mask & (~(size_t)movemask(terminated));
|
|
|
|
if (unlikely(m_active == 0)) continue;
|
|
|
|
while (likely(!cur.isLeaf()))
|
|
{
|
|
/* process nodes */
|
|
//STAT3(normal.trav_nodes, 1, popcnt(valid_node), K);
|
|
const NodeRef nodeRef = cur;
|
|
const AABBNode* __restrict__ const node = nodeRef.getAABBNode();
|
|
|
|
vfloat<N> fmin;
|
|
size_t m_frustum_node = intersectNodeFrustum<N>(node, frustum, fmin);
|
|
|
|
if (unlikely(!m_frustum_node)) goto pop;
|
|
cur = BVH::emptyNode;
|
|
m_active = 0;
|
|
|
|
#if defined(__AVX__)
|
|
//STAT3(normal.trav_hit_boxes[popcnt(m_frustum_node)], 1, 1, 1);
|
|
#endif
|
|
//size_t num_child_hits = 0;
|
|
do {
|
|
const size_t i = bscf(m_frustum_node);
|
|
vfloat<K> lnearP;
|
|
vbool<K> lhit = false; // motion blur is not supported, so the initial value will be ignored
|
|
STAT3(normal.trav_nodes, 1, 1, 1);
|
|
BVHNNodeIntersectorK<N, K, types, robust>::intersect(nodeRef, i, tray, ray.time(), lnearP, lhit);
|
|
|
|
if (likely(any(lhit)))
|
|
{
|
|
const NodeRef child = node->child(i);
|
|
assert(child != BVH::emptyNode);
|
|
BVHN<N>::prefetch(child);
|
|
if (likely(cur != BVH::emptyNode)) {
|
|
//num_child_hits++;
|
|
stackPtr->ptr = cur;
|
|
stackPtr->mask = m_active;
|
|
stackPtr++;
|
|
}
|
|
cur = child;
|
|
m_active = movemask(lhit);
|
|
}
|
|
} while(m_frustum_node);
|
|
|
|
if (unlikely(cur == BVH::emptyNode)) goto pop;
|
|
}
|
|
|
|
/* intersect leaf */
|
|
assert(cur != BVH::invalidNode);
|
|
assert(cur != BVH::emptyNode);
|
|
#if defined(__AVX__)
|
|
STAT3(normal.trav_leaves, 1, popcnt(m_active), K);
|
|
#endif
|
|
if (unlikely(!m_active)) continue;
|
|
size_t items; const Primitive* prim = (Primitive*)cur.leaf(items);
|
|
|
|
size_t lazy_node = 0;
|
|
terminated |= PrimitiveIntersectorK::occluded(!terminated, This, pre, ray, context, prim, items, tray, lazy_node);
|
|
octant_valid &= !terminated;
|
|
if (unlikely(none(octant_valid))) break;
|
|
tray.tfar = select(terminated, vfloat<K>(neg_inf), tray.tfar); // ignore node intersections for terminated rays
|
|
|
|
if (unlikely(lazy_node)) {
|
|
stackPtr->ptr = lazy_node;
|
|
stackPtr->mask = movemask(octant_valid);
|
|
stackPtr++;
|
|
}
|
|
}
|
|
} while(valid_bits);
|
|
|
|
vfloat<K>::store(valid & terminated, &ray.tfar, neg_inf);
|
|
}
|
|
}
|
|
}
|