godotengine
/
godot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
godot/thirdparty/embree/kernels/bvh/bvh_intersector_stream.cpp

529 lines
22 KiB
C++
Raw Blame History

// Copyright 2009-2021 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
#include "bvh_intersector_stream.h"
#include "../geometry/intersector_iterators.h"
#include "../geometry/triangle_intersector.h"
#include "../geometry/trianglev_intersector.h"
#include "../geometry/trianglev_mb_intersector.h"
#include "../geometry/trianglei_intersector.h"
#include "../geometry/quadv_intersector.h"
#include "../geometry/quadi_intersector.h"
#include "../geometry/linei_intersector.h"
#include "../geometry/subdivpatch1_intersector.h"
#include "../geometry/object_intersector.h"
#include "../geometry/instance_intersector.h"
#include "../common/scene.h"
#include <bitset>
namespace embree
{
namespace isa
{
__aligned(64) static const int shiftTable[32] = {
(int)1 << 0, (int)1 << 1, (int)1 << 2, (int)1 << 3, (int)1 << 4, (int)1 << 5, (int)1 << 6, (int)1 << 7,
(int)1 << 8, (int)1 << 9, (int)1 << 10, (int)1 << 11, (int)1 << 12, (int)1 << 13, (int)1 << 14, (int)1 << 15,
(int)1 << 16, (int)1 << 17, (int)1 << 18, (int)1 << 19, (int)1 << 20, (int)1 << 21, (int)1 << 22, (int)1 << 23,
(int)1 << 24, (int)1 << 25, (int)1 << 26, (int)1 << 27, (int)1 << 28, (int)1 << 29, (int)1 << 30, (int)1 << 31
};
template<int N, int types, bool robust, typename PrimitiveIntersector>
__forceinline void BVHNIntersectorStream<N, types, robust, PrimitiveIntersector>::intersect(Accel::Intersectors* __restrict__ This,
RayHitN** inputPackets,
size_t numOctantRays,
IntersectContext* context)
{
/* we may traverse an empty BVH in case all geometry was invalid */
BVH* __restrict__ bvh = (BVH*) This->ptr;
if (bvh->root == BVH::emptyNode)
return;
// Only the coherent code path is implemented
assert(context->isCoherent());
intersectCoherent(This, (RayHitK<VSIZEL>**)inputPackets, numOctantRays, context);
}
template<int N, int types, bool robust, typename PrimitiveIntersector>
template<int K>
__forceinline void BVHNIntersectorStream<N, types, robust, PrimitiveIntersector>::intersectCoherent(Accel::Intersectors* __restrict__ This,
RayHitK<K>** inputPackets,
size_t numOctantRays,
IntersectContext* context)
{
assert(context->isCoherent());
BVH* __restrict__ bvh = (BVH*) This->ptr;
__aligned(64) StackItemMaskCoherent stack[stackSizeSingle]; // stack of nodes
assert(numOctantRays <= MAX_INTERNAL_STREAM_SIZE);
__aligned(64) TravRayKStream<K, robust> packets[MAX_INTERNAL_STREAM_SIZE/K];
__aligned(64) Frustum<robust> frustum;
bool commonOctant = true;
const size_t m_active = initPacketsAndFrustum((RayK<K>**)inputPackets, numOctantRays, packets, frustum, commonOctant);
if (unlikely(m_active == 0)) return;
/* case of non-common origin */
if (unlikely(!commonOctant))
{
const size_t numPackets = (numOctantRays+K-1)/K;
for (size_t i = 0; i < numPackets; i++)
This->intersect(inputPackets[i]->tnear() <= inputPackets[i]->tfar, *inputPackets[i], context);
return;
}
stack[0].mask = m_active;
stack[0].parent = 0;
stack[0].child = bvh->root;
///////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////
StackItemMaskCoherent* stackPtr = stack + 1;
while (1) pop:
{
if (unlikely(stackPtr == stack)) break;
STAT3(normal.trav_stack_pop,1,1,1);
stackPtr--;
/*! pop next node */
NodeRef cur = NodeRef(stackPtr->child);
size_t m_trav_active = stackPtr->mask;
assert(m_trav_active);
NodeRef parent = stackPtr->parent;
while (1)
{
if (unlikely(cur.isLeaf())) break;
const AABBNode* __restrict__ const node = cur.getAABBNode();
parent = cur;
__aligned(64) size_t maskK[N];
for (size_t i = 0; i < N; i++)
maskK[i] = m_trav_active;
vfloat<N> dist;
const size_t m_node_hit = traverseCoherentStream(m_trav_active, packets, node, frustum, maskK, dist);
if (unlikely(m_node_hit == 0)) goto pop;
BVHNNodeTraverserStreamHitCoherent<N, types>::traverseClosestHit(cur, m_trav_active, vbool<N>((int)m_node_hit), dist, (size_t*)maskK, stackPtr);
assert(m_trav_active);
}
/* non-root and leaf => full culling test for all rays */
if (unlikely(parent != 0 && cur.isLeaf()))
{
const AABBNode* __restrict__ const node = parent.getAABBNode();
size_t boxID = 0xff;
for (size_t i = 0; i < N; i++)
if (node->child(i) == cur) { boxID = i; break; }
assert(boxID < N);
assert(cur == node->child(boxID));
m_trav_active = intersectAABBNodePacket(m_trav_active, packets, node, boxID, frustum.nf);
}
/*! this is a leaf node */
assert(cur != BVH::emptyNode);
STAT3(normal.trav_leaves, 1, 1, 1);
size_t num; PrimitiveK<K>* prim = (PrimitiveK<K>*)cur.leaf(num);
size_t bits = m_trav_active;
/*! intersect stream of rays with all primitives */
size_t lazy_node = 0;
#if defined(__SSE4_2__)
STAT_USER(1,(popcnt(bits)+K-1)/K*4);
#endif
while(bits)
{
size_t i = bsf(bits) / K;
const size_t m_isec = ((((size_t)1 << K)-1) << (i*K));
assert(m_isec & bits);
bits &= ~m_isec;
TravRayKStream<K, robust>& p = packets[i];
vbool<K> m_valid = p.tnear <= p.tfar;
PrimitiveIntersectorK<K>::intersectK(m_valid, This, *inputPackets[i], context, prim, num, lazy_node);
p.tfar = min(p.tfar, inputPackets[i]->tfar);
};
} // traversal + intersection
}
template<int N, int types, bool robust, typename PrimitiveIntersector>
__forceinline void BVHNIntersectorStream<N, types, robust, PrimitiveIntersector>::occluded(Accel::Intersectors* __restrict__ This,
RayN** inputPackets,
size_t numOctantRays,
IntersectContext* context)
{
/* we may traverse an empty BVH in case all geometry was invalid */
BVH* __restrict__ bvh = (BVH*) This->ptr;
if (bvh->root == BVH::emptyNode)
return;
if (unlikely(context->isCoherent()))
occludedCoherent(This, (RayK<VSIZEL>**)inputPackets, numOctantRays, context);
else
occludedIncoherent(This, (RayK<VSIZEX>**)inputPackets, numOctantRays, context);
}
template<int N, int types, bool robust, typename PrimitiveIntersector>
template<int K>
__noinline void BVHNIntersectorStream<N, types, robust, PrimitiveIntersector>::occludedCoherent(Accel::Intersectors* __restrict__ This,
RayK<K>** inputPackets,
size_t numOctantRays,
IntersectContext* context)
{
assert(context->isCoherent());
BVH* __restrict__ bvh = (BVH*)This->ptr;
__aligned(64) StackItemMaskCoherent stack[stackSizeSingle]; // stack of nodes
assert(numOctantRays <= MAX_INTERNAL_STREAM_SIZE);
/* inactive rays should have been filtered out before */
__aligned(64) TravRayKStream<K, robust> packets[MAX_INTERNAL_STREAM_SIZE/K];
__aligned(64) Frustum<robust> frustum;
bool commonOctant = true;
size_t m_active = initPacketsAndFrustum(inputPackets, numOctantRays, packets, frustum, commonOctant);
/* valid rays */
if (unlikely(m_active == 0)) return;
/* case of non-common origin */
if (unlikely(!commonOctant))
{
const size_t numPackets = (numOctantRays+K-1)/K;
for (size_t i = 0; i < numPackets; i++)
This->occluded(inputPackets[i]->tnear() <= inputPackets[i]->tfar, *inputPackets[i], context);
return;
}
stack[0].mask = m_active;
stack[0].parent = 0;
stack[0].child = bvh->root;
///////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////
StackItemMaskCoherent* stackPtr = stack + 1;
while (1) pop:
{
if (unlikely(stackPtr == stack)) break;
STAT3(normal.trav_stack_pop,1,1,1);
stackPtr--;
/*! pop next node */
NodeRef cur = NodeRef(stackPtr->child);
size_t m_trav_active = stackPtr->mask & m_active;
if (unlikely(!m_trav_active)) continue;
assert(m_trav_active);
NodeRef parent = stackPtr->parent;
while (1)
{
if (unlikely(cur.isLeaf())) break;
const AABBNode* __restrict__ const node = cur.getAABBNode();
parent = cur;
__aligned(64) size_t maskK[N];
for (size_t i = 0; i < N; i++)
maskK[i] = m_trav_active;
vfloat<N> dist;
const size_t m_node_hit = traverseCoherentStream(m_trav_active, packets, node, frustum, maskK, dist);
if (unlikely(m_node_hit == 0)) goto pop;
BVHNNodeTraverserStreamHitCoherent<N, types>::traverseAnyHit(cur, m_trav_active, vbool<N>((int)m_node_hit), (size_t*)maskK, stackPtr);
assert(m_trav_active);
}
/* non-root and leaf => full culling test for all rays */
if (unlikely(parent != 0 && cur.isLeaf()))
{
const AABBNode* __restrict__ const node = parent.getAABBNode();
size_t boxID = 0xff;
for (size_t i = 0; i < N; i++)
if (node->child(i) == cur) { boxID = i; break; }
assert(boxID < N);
assert(cur == node->child(boxID));
m_trav_active = intersectAABBNodePacket(m_trav_active, packets, node, boxID, frustum.nf);
}
/*! this is a leaf node */
assert(cur != BVH::emptyNode);
STAT3(normal.trav_leaves, 1, 1, 1);
size_t num; PrimitiveK<K>* prim = (PrimitiveK<K>*)cur.leaf(num);
size_t bits = m_trav_active & m_active;
/*! intersect stream of rays with all primitives */
size_t lazy_node = 0;
#if defined(__SSE4_2__)
STAT_USER(1,(popcnt(bits)+K-1)/K*4);
#endif
while (bits)
{
size_t i = bsf(bits) / K;
const size_t m_isec = ((((size_t)1 << K)-1) << (i*K));
assert(m_isec & bits);
bits &= ~m_isec;
TravRayKStream<K, robust>& p = packets[i];
vbool<K> m_valid = p.tnear <= p.tfar;
vbool<K> m_hit = PrimitiveIntersectorK<K>::occludedK(m_valid, This, *inputPackets[i], context, prim, num, lazy_node);
inputPackets[i]->tfar = select(m_hit & m_valid, vfloat<K>(neg_inf), inputPackets[i]->tfar);
m_active &= ~((size_t)movemask(m_hit) << (i*K));
}
} // traversal + intersection
}
template<int N, int types, bool robust, typename PrimitiveIntersector>
template<int K>
__forceinline void BVHNIntersectorStream<N, types, robust, PrimitiveIntersector>::occludedIncoherent(Accel::Intersectors* __restrict__ This,
RayK<K>** inputPackets,
size_t numOctantRays,
IntersectContext* context)
{
assert(!context->isCoherent());
assert(types & BVH_FLAG_ALIGNED_NODE);
__aligned(64) TravRayKStream<K,robust> packet[MAX_INTERNAL_STREAM_SIZE/K];
assert(numOctantRays <= 32);
const size_t numPackets = (numOctantRays+K-1)/K;
size_t m_active = 0;
for (size_t i = 0; i < numPackets; i++)
{
const vfloat<K> tnear = inputPackets[i]->tnear();
const vfloat<K> tfar = inputPackets[i]->tfar;
vbool<K> m_valid = (tnear <= tfar) & (tnear >= 0.0f);
m_active |= (size_t)movemask(m_valid) << (K*i);
const Vec3vf<K>& org = inputPackets[i]->org;
const Vec3vf<K>& dir = inputPackets[i]->dir;
vfloat<K> packet_min_dist = max(tnear, 0.0f);
vfloat<K> packet_max_dist = select(m_valid, tfar, neg_inf);
new (&packet[i]) TravRayKStream<K,robust>(org, dir, packet_min_dist, packet_max_dist);
}
BVH* __restrict__ bvh = (BVH*)This->ptr;
StackItemMaskT<NodeRef> stack[stackSizeSingle]; // stack of nodes
StackItemMaskT<NodeRef>* stackPtr = stack + 1; // current stack pointer
stack[0].ptr = bvh->root;
stack[0].mask = m_active;
size_t terminated = ~m_active;
/* near/far offsets based on first ray */
const NearFarPrecalculations nf(Vec3fa(packet[0].rdir.x[0], packet[0].rdir.y[0], packet[0].rdir.z[0]), N);
while (1) pop:
{
if (unlikely(stackPtr == stack)) break;
STAT3(shadow.trav_stack_pop,1,1,1);
stackPtr--;
NodeRef cur = NodeRef(stackPtr->ptr);
size_t cur_mask = stackPtr->mask & (~terminated);
if (unlikely(cur_mask == 0)) continue;
while (true)
{
/*! stop if we found a leaf node */
if (unlikely(cur.isLeaf())) break;
const AABBNode* __restrict__ const node = cur.getAABBNode();
const vint<N> vmask = traverseIncoherentStream(cur_mask, packet, node, nf, shiftTable);
size_t mask = movemask(vmask != vint<N>(zero));
if (unlikely(mask == 0)) goto pop;
__aligned(64) unsigned int child_mask[N];
vint<N>::storeu(child_mask, vmask); // this explicit store here causes much better code generation
/*! one child is hit, continue with that child */
size_t r = bscf(mask);
assert(r < N);
cur = node->child(r);
BVHN<N>::prefetch(cur,types);
cur_mask = child_mask[r];
/* simple in order sequence */
assert(cur != BVH::emptyNode);
if (likely(mask == 0)) continue;
stackPtr->ptr = cur;
stackPtr->mask = cur_mask;
stackPtr++;
for (; ;)
{
r = bscf(mask);
assert(r < N);
cur = node->child(r);
BVHN<N>::prefetch(cur,types);
cur_mask = child_mask[r];
assert(cur != BVH::emptyNode);
if (likely(mask == 0)) break;
stackPtr->ptr = cur;
stackPtr->mask = cur_mask;
stackPtr++;
}
}
/*! this is a leaf node */
assert(cur != BVH::emptyNode);
STAT3(shadow.trav_leaves,1,1,1);
size_t num; PrimitiveK<K>* prim = (PrimitiveK<K>*)cur.leaf(num);
size_t bits = cur_mask;
size_t lazy_node = 0;
for (; bits != 0;)
{
const size_t rayID = bscf(bits);
RayK<K> &ray = *inputPackets[rayID / K];
const size_t k = rayID % K;
if (PrimitiveIntersectorK<K>::occluded(This, ray, k, context, prim, num, lazy_node))
{
ray.tfar[k] = neg_inf;
terminated |= (size_t)1 << rayID;
}
/* lazy node */
if (unlikely(lazy_node))
{
stackPtr->ptr = lazy_node;
stackPtr->mask = cur_mask;
stackPtr++;
}
}
if (unlikely(terminated == (size_t)-1)) break;
}
}
////////////////////////////////////////////////////////////////////////////////
/// ArrayIntersectorKStream Definitions
////////////////////////////////////////////////////////////////////////////////
template<bool filter>
struct Triangle4IntersectorStreamMoeller {
template<int K> using Type = ArrayIntersectorKStream<K,TriangleMIntersectorKMoeller<4 COMMA K COMMA true>>;
};
template<bool filter>
struct Triangle4vIntersectorStreamPluecker {
template<int K> using Type = ArrayIntersectorKStream<K,TriangleMvIntersectorKPluecker<4 COMMA K COMMA true>>;
};
template<bool filter>
struct Triangle4iIntersectorStreamMoeller {
template<int K> using Type = ArrayIntersectorKStream<K,TriangleMiIntersectorKMoeller<4 COMMA K COMMA true>>;
};
template<bool filter>
struct Triangle4iIntersectorStreamPluecker {
template<int K> using Type = ArrayIntersectorKStream<K,TriangleMiIntersectorKPluecker<4 COMMA K COMMA true>>;
};
template<bool filter>
struct Quad4vIntersectorStreamMoeller {
template<int K> using Type = ArrayIntersectorKStream<K,QuadMvIntersectorKMoeller<4 COMMA K COMMA true>>;
};
template<bool filter>
struct Quad4iIntersectorStreamMoeller {
template<int K> using Type = ArrayIntersectorKStream<K,QuadMiIntersectorKMoeller<4 COMMA K COMMA true>>;
};
template<bool filter>
struct Quad4vIntersectorStreamPluecker {
template<int K> using Type = ArrayIntersectorKStream<K,QuadMvIntersectorKPluecker<4 COMMA K COMMA true>>;
};
template<bool filter>
struct Quad4iIntersectorStreamPluecker {
template<int K> using Type = ArrayIntersectorKStream<K,QuadMiIntersectorKPluecker<4 COMMA K COMMA true>>;
};
struct ObjectIntersectorStream {
template<int K> using Type = ArrayIntersectorKStream<K,ObjectIntersectorK<K COMMA false>>;
};
struct InstanceIntersectorStream {
template<int K> using Type = ArrayIntersectorKStream<K,InstanceIntersectorK<K>>;
};
// =====================================================================================================
// =====================================================================================================
// =====================================================================================================
template<int N>
void BVHNIntersectorStreamPacketFallback<N>::intersect(Accel::Intersectors* __restrict__ This,
RayHitN** inputRays,
size_t numTotalRays,
IntersectContext* context)
{
if (unlikely(context->isCoherent()))
intersectK(This, (RayHitK<VSIZEL>**)inputRays, numTotalRays, context);
else
intersectK(This, (RayHitK<VSIZEX>**)inputRays, numTotalRays, context);
}
template<int N>
void BVHNIntersectorStreamPacketFallback<N>::occluded(Accel::Intersectors* __restrict__ This,
RayN** inputRays,
size_t numTotalRays,
IntersectContext* context)
{
if (unlikely(context->isCoherent()))
occludedK(This, (RayK<VSIZEL>**)inputRays, numTotalRays, context);
else
occludedK(This, (RayK<VSIZEX>**)inputRays, numTotalRays, context);
}
template<int N>
template<int K>
__noinline void BVHNIntersectorStreamPacketFallback<N>::intersectK(Accel::Intersectors* __restrict__ This,
RayHitK<K>** inputRays,
size_t numTotalRays,
IntersectContext* context)
{
/* fallback to packets */
for (size_t i = 0; i < numTotalRays; i += K)
{
const vint<K> vi = vint<K>(int(i)) + vint<K>(step);
vbool<K> valid = vi < vint<K>(int(numTotalRays));
RayHitK<K>& ray = *(inputRays[i / K]);
valid &= ray.tnear() <= ray.tfar;
This->intersect(valid, ray, context);
}
}
template<int N>
template<int K>
__noinline void BVHNIntersectorStreamPacketFallback<N>::occludedK(Accel::Intersectors* __restrict__ This,
RayK<K>** inputRays,
size_t numTotalRays,
IntersectContext* context)
{
/* fallback to packets */
for (size_t i = 0; i < numTotalRays; i += K)
{
const vint<K> vi = vint<K>(int(i)) + vint<K>(step);
vbool<K> valid = vi < vint<K>(int(numTotalRays));
RayK<K>& ray = *(inputRays[i / K]);
valid &= ray.tnear() <= ray.tfar;
This->occluded(valid, ray, context);
}
}
}
}
Reference in New Issue View Git Blame Copy Permalink