error X4532 Cannot map expression to gs_5_0 instruction set

Question

Using DX11 I have this line starting with TC = txDiffuse in a geometry shader that generates the error message. Can someone explain me the problem or if it is not possible to sample in geometry shader (gs_5 model).

 [maxvertexcount(6)]
 void GS_Deferred( triangle GS_INPUT input[3], inout TriangleStream<PS_DEFERRED> CubeMapStream )
 {
     PS_DEFERRED output;
     for( int f = 0; f < 2; ++f )
     {
         output.RTIndex = f;
         for( int v = 0; v < 3; v++ )
         {
             GS_INPUT I = input[v];
             if ( f==0 )
             {
                 float3 TC = float3(1,1,1);;
    ------>      TC = txDiffuse.SampleLevel(samLinear, I.Tex, 0);//same with Sample()
                 output.Color = float4(I.Col.rgb*TC, 1);
             }
             if ( f==1 )
             {
                float3 NBump = I.Norm;
                if ( Flag & BUMPNORM )
                {
                    float3x3 BTNMatrix = float3x3(I.BiN,  I.Tan, I.Norm);
                    NBump = normalize(mul(txNormal.Sample(samPoint, I.Tex).rgb*2 - 1,BTNMatrix));
                }
                output.Color = float4(NBump*0.5+0.5,1);
             }
             output.Pos = I.Pos;
             CubeMapStream.Append( output );
          }
          CubeMapStream.RestartStrip();
      }
  }

We'd expect this error for Sample, which is not supported in the geometry shader, since it wants to use screenspace derivative information to auto-select the right mip level/anisotropic filtering, and those derivatives are only available in the pixel shader. But this should work for SampleLevel. I note you have a double-semicolon on the line above, is that related? — DMGregory, Oct 13 '22 at 15:30
I've corrected the typo, checked GSResourceShader are well-updated, and I have a first result with is the raw vertex colors in target 0 (f=0) if TC = 1 otherwise if TC =texture everything is black. Also Target1 (f=1) is full black where it should have the normals. I'm working on that. — philB, Oct 13 '22 at 16:02

philB · Accepted Answer · 2022-10-14T14:26:32.233

After having corrected a typo and a couple of other things, I get something on screen but I realized that in fact as vertex shaders have always done, geometry shaders are just outputing the vertex color that is then interpolated in the pixel shader. Indeed I can't expect the geometry shader to produce perpixel interpolation required for using a texture as material. So I moved this sampling to the pixel shader where things are correctly done.

Now regarding the behavior of my initial geometry shader, a couple of settings and CPU coding were missing to have it working properly.

For instance I learned on the way that a rendertargetarray needs the corresponding depthstencilarray if you need depth test. Finally I get what I was expecting: objects being displayed in selected layers of my 4 layers of data used for screen space reflection as colors, normals and depth (in this latter case I have twice the same Zbuffer so an optimization is needed). See picture.

Below the modified geometry shader doing all this.

cbuffer cbMesh : register(b1)
{
    matrix World;
    matrix IT_VW;
    float4 FactorColor;
    float4  VisibilityLayer;//xyzw = O/1. the key param for the drawing on selected layers. x=layer 0 .. w=layer 3
    dword   Flag;//adding some object specific shading
    float3 pad;
}
struct VS_INPUT
{
    float3 Pos : POSITION;
    float3 Norm : NORMAL;
    float4 Col : COLOR0;
    float2 Tex : TEXCOORD0;
};
struct GS_INPUT
{
    float4 Pos : SV_POSITION;
    float3 Norm : NORMAL;
    float4 Col : COLOR0;
    float2 Tex : TEXCOORD0;
    float3 Tan : TANGENT;
    float3 BiN : BINORMAL;
};
struct PS_DEFERRED
{
    float4 Pos : SV_POSITION;
    float4 Color0 : COLOR0;
    float2 Tex : TEXCOORD0;
    float3 Tan : TANGENT;
    float3 BiN : BINORMAL;
    uint RTIndex : SV_RenderTargetArrayIndex;
};
GS_INPUT VS_Deferred( VS_INPUT input )
{
    all classical calculs here
}
[maxvertexcount(24)]
void GS_Deferred( triangle GS_INPUT input[3], inout TriangleStream<PS_DEFERRED> CubeMapStream )
{
    for ( int f=0;f<8;++f)
    {
        PS_DEFERRED O;
        float Visible = 1;
        O.RTIndex = f;
        //f=0, 4 always all visible (e.g. VisibilityLayer.x always 1)
        if (( f==1 )|| (f==5)) Visible = VisibilityLayer.y;
        if (( f==2 )|| (f==6)) Visible = VisibilityLayer.z;
        if (( f==3 )|| (f==7)) Visible = VisibilityLayer.w;
        if ( Visible )
        {
            for( int v = 0; v < 3; v++ )
            {
                O.Tex = input[v].Tex;
                O.Pos = input[v].Pos;
                O.BiN = input[v].BiN;
                O.Tan = input[v].Tan;
                O.Color0 = ( f < 4)?input[v].Col:float4(input[v].Norm,1);
                CubeMapStream.Append( O );
            }
            CubeMapStream.RestartStrip();
        }
    }
}
float4 PS_Deferred(PS_DEFERRED Input): SV_TARGET
{
    float4 Color = Input.Color0;
    if (Input.RTIndex < 4)
    {
        float3 TC = ( Flag & dwColorTex0)?txDiffuse.SampleLevel(samLinear, Input.Tex, 0).rgb:float3(1,1,1);
        Color = float4(Color.xyzTC,Color.a);
    }
    else
    {
        if ( Flag & BUMPNORM )
        {
            float3x3 BTNMatrix = float3x3(Input.BiN,  Input.Tan, Color.xyz);
            Color.xyz = normalize(mul(txNormal.Sample(samPoint, Input.Tex).rgb2 - 1,BTNMatrix));
        }
        Color.xyz = Color.xyz*0.5+0.5;
    }
    return Color;
}

Remember to mark this answer as Accepted if it solved your problem, once the self-answer waiting period is up. — DMGregory, Oct 14 '22 at 17:56

error X4532 Cannot map expression to gs_5_0 instruction set

1 Answers1