androidopengl-eslibgdxstencil-buffer

LibGDX / OpenGL Stencil Buffer masking not working


I'm using LibGDX to render 2d fog-of-war type functionality. This involves drawing a dark rectangle over the entire map with transparent holes in it where you can see the map below. I'm attempting to use OpenGl stencil buffer to create the circular masks, but I can't seem to get the logic correct.

The code below correctly draws the dark rectangle (the fog) but the circular masks are not being stenciled. i.e. the entire map is dark.

 batch.end();

    Gdx.gl.glClear(GL_STENCIL_BUFFER_BIT);
    Gdx.gl.glEnable(GL20.GL_STENCIL_TEST);
    Gdx.gl.glColorMask(false, false, false, false);
    //always write the clipped holes to the stencil
    Gdx.gl.glStencilFunc(GL20.GL_ALWAYS, 0x1, 0xffffffff);
    Gdx.gl.glStencilMask(0xFF);
    Gdx.gl.glStencilOp(GL_REPLACE, GL_REPLACE, GL_REPLACE);

    shapeRenderer.begin(ShapeRenderer.ShapeType.Filled);
    shapeRenderer.setProjectionMatrix(gameUi.camera.combined);
   
     //test circle
    shapeRenderer.circle(0, 0, 1000, 100);

    shapeRenderer.end();


    Gdx.gl.glEnable(GL20.GL_BLEND);
    Gdx.gl.glBlendFunc(GL20.GL_SRC_ALPHA, GL20.GL_ONE_MINUS_SRC_ALPHA);
    Gdx.gl.glColorMask(true, true, true, true);
    Gdx.gl.glEnable(GL20.GL_STENCIL_TEST);
    //only draw the fog of war where stencil is 0
    Gdx.gl.glStencilFunc(GL_EQUAL, 0x0, 0xffffffff);
    Gdx.gl.glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);


    //draw fog of war
    shapeRenderer.begin(ShapeRenderer.ShapeType.Filled);
    shapeRenderer.setProjectionMatrix(gameUi.camera.combined);
    shapeRenderer.setColor(radarFog);
    int dimension = gameUi.mapConfig.getDimension();
    shapeRenderer.rect(-dimension,-dimension,dimension*3,dimension*3);
    shapeRenderer.end();

    Gdx.gl.glDisable(GL20.GL_STENCIL_TEST);

    batch.begin();

Solution

  • I was finally able to get this to work with stencils. There were some issues with the stencil ops I was using but the code below is working for me.

            //enable stencil and depth testing
            gl.glEnable(GL_STENCIL_TEST);
            gl.glEnable(GL_DEPTH_TEST);
            //enable blending because our fog has alpha
            Gdx.gl.glEnable(GL20.GL_BLEND);
            Gdx.gl.glBlendFunc(GL20.GL_SRC_ALPHA, GL20.GL_ONE_MINUS_SRC_ALPHA);
            //stencil op which writes to stencil when test passes
            gl.glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
            //clear stencil buffer
            gl.glClear(GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
            //always pass the stencil test and mask to accept all values
            gl.glStencilFunc(GL_ALWAYS, 1, 0xFF);
            gl.glStencilMask(0xFF);
    
            //draw our cut out circles
            drawCircles();
    
            //change stencil function to only match where stencil was written
            gl.glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
            //change mask so nothing is written to stencil
            gl.glStencilMask(0x00);
    
    
            drawFog();
    
            //disable features
            gl.glDisable(GL_STENCIL_TEST);
            gl.glDisable(GL_DEPTH_TEST);
            gl.glDisable(GL_BLEND);