libgdxmaskingstencil-bufferdecal

libgdx decal not drawing into stencil buffer


I'm working on 3d app in libgdx engine.

I just figured out that decalBatch isn't drawing into stencil buffer. I wanned to make stencil masks for 3d world, and it's not working at all.

This is the code which works for sprite batch, but it's not working with decal batch. Pls help!

    Gdx.gl.glClearColor(1, 1, 1, 1);
    Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT | GL20.GL_STENCIL_BUFFER_BIT);
    // batch.setProjectionMatrix(camera.combined);

    // setup drawing to stencil buffer
    Gdx.gl20.glEnable(GL20.GL_STENCIL_TEST);
    Gdx.gl20.glStencilFunc(GL20.GL_ALWAYS, 0x1, 0xffffffff);
    Gdx.gl20.glStencilOp(GL20.GL_REPLACE, GL20.GL_REPLACE, GL20.GL_REPLACE);
    Gdx.gl20.glColorMask(false, false, false, false);

    // draw base pattern
    shapeRenderer.begin(ShapeType.Filled);
    shapeRenderer.identity();
    shapeRenderer.setColor(1f, 0f, 0f, 0.5f);

    shapeRenderer.rect(Gdx.graphics.getWidth(), Gdx.graphics.getHeight(), 100, 100);

    shapeRenderer.end();

    spriteBatch.begin();
    // fix stencil buffer, enable color buffer
    Gdx.gl20.glColorMask(true, true, true, true);
    Gdx.gl20.glStencilOp(GL20.GL_KEEP, GL20.GL_KEEP, GL20.GL_KEEP);

    // draw where pattern has NOT been drawn
    Gdx.gl20.glStencilFunc(GL20.GL_NOTEQUAL, 0x1, 0xff);

    // decalBatch.add(decal);
    // decalBatch.flush();

    spriteBatch.draw(Assets.instance.background, 0, 0, Gdx.graphics.getWidth(),         Gdx.graphics.getHeight());

    // draw where pattern HAS been drawn.
    // Gdx.gl20.glStencilFunc(GL20.GL_EQUAL, 0x1, 0xff);

    // spriteBatch.draw(Assets.instance.actor1, -Gdx.graphics.getWidth() /
    // 2, -Gdx.graphics.getHeight() / 2, Gdx.graphics.getWidth(),
    // Gdx.graphics.getHeight());

    spriteBatch.end();

EDIT:

I figured out that there should be clearing of Depth buffer, and enabling and disabling of DepthMask, but I cant manage it to work.


Solution

  • Oh I just figured it out. Depth buffer must be enabled couse Decals are in 3d world.

    this is the solution if somebody is interested:

        Gdx.gl.glClearColor(1, 1, 1, 1);
        Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT | GL20.GL_STENCIL_BUFFER_BIT |     GL20.GL_DEPTH_BUFFER_BIT);
    
        // setup drawing to stencil buffer
        Gdx.gl20.glEnable(GL20.GL_STENCIL_TEST);
        Gdx.gl20.glStencilFunc(GL20.GL_ALWAYS, 0x1, 0xffffffff);
        Gdx.gl20.glStencilOp(GL20.GL_REPLACE, GL20.GL_REPLACE, GL20.GL_REPLACE);
        Gdx.gl20.glColorMask(false, false, false, false);
        Gdx.gl20.glDepthMask(false);
    
        // draw base pattern
        shapeRenderer.begin(ShapeType.Filled);
        shapeRenderer.identity();
        shapeRenderer.setColor(1f, 0f, 0f, 0.5f);
    
        shapeRenderer.rect(Gdx.graphics.getWidth() / 2, Gdx.graphics.getHeight() / 2, 100, 100);
    
        shapeRenderer.end();
    
        // fix stencil buffer, enable color buffer
        Gdx.gl20.glColorMask(true, true, true, true);
        Gdx.gl20.glDepthMask(true);
        Gdx.gl20.glStencilOp(GL20.GL_KEEP, GL20.GL_KEEP, GL20.GL_KEEP);
    
        // draw where pattern has NOT been drawn
        Gdx.gl20.glStencilFunc(GL20.GL_NOTEQUAL, 0x1, 0xff);
    
        decalBatch.add(decal);
        decalBatch.flush();
    
        Gdx.gl20.glDisable(GL20.GL_STENCIL_TEST);