Pixi Filter/Shader with Confusing Results

Blackdrazon · November 22, 2017

I'm experimenting with GLSL shaders in Pixi 4.4, and was trying to make some that would take in two images, the base and an overlay. The shaders would then replace either the Hue, Saturation, or Value of the pixels in the base image with the H/S/V of the corresponding pixel in the overlay image. Transparency on the overlay means "no change."

For my tests, I used a 100x100 red square, and the following 100x100 "striped" overlays:

stripesHue.png.0be7f60b48964b955397daf626de4ac5.png stripesSat.png.a5dca7f672ee175308bf589c741e2373.png stripesVal.png.d014039e15eeb9729ed1d9e11eac17d6.png

That's the Hue overlay, Saturation overlay, and Value overlay respectively.

Results were only partially consistent, and all wrong. Here are the results of the Hue and Saturation shaders against a black background.

Screenshot_10.png.b1c030900f3a09b84f40f901e78884f0.png Screenshot_11.png.f92646dda8f259454134faf1016eafe0.png

Okay, so the stripes are twice as tall as they should be and the bottom-most stripe is outright missing, as though either the overlay or base were resized as some point in the process. But Value takes the cake:

Screenshot_12.png.c86ad4dd40fb5d0b24f613feb82fb0ca.png

Not only do we have the same problem as above, but there are "teeth" that have appeared off the edge of the 100x100 image (4px in each direction, making a 108x100 image), and there are outlines around every stripe, and if you zoom in especially close you can see that some of the outlines are actually 2 pixels tall, one of near-black, and one of another dark colour, none of which is in the original Value overlay!

I'm at a loss to tell if the problem(s) originates in my shader code or in Pixi, especially since tutorials around the net are mum about how to create a second shader2D uniform in any other setup but Pixi. I do want to work with Pixi for this project, however, so a fix for Pixi would be appreciated if the problem really is from there.

Here's the HTML/GLSL code. Please don't mind the If statement, I've already had a few ideas on how to get rid of it:

<html>
<head>
<meta content="text/html;charset=utf-8" http-equiv="Content-Type">
<meta content="utf-8" http-equiv="encoding">

<style>
body {
  background-color: black;
  margin: 0;
  overflow: hidden;
}

p {
  color: white;
}
</style>

</head>

<body>

    <script type="text/javascript" src="libs/pixi.js"></script>
	
	<script id="shader" type="shader">
		#ifdef GL_ES
		precision mediump float;
		#endif

		varying vec2 vTextureCoord;
		uniform sampler2D uSampler; //The base image
		uniform sampler2D overlay; //The overlay

		vec3 rgb2hsv(vec3 c)
		{
			vec4 K = vec4(0.0, -1.0 / 3.0, 2.0 / 3.0, -1.0);
			vec4 p = mix(vec4(c.bg, K.wz), vec4(c.gb, K.xy), step(c.b, c.g));
			vec4 q = mix(vec4(p.xyw, c.r), vec4(c.r, p.yzx), step(p.x, c.r));

			float d = q.x - min(q.w, q.y);
			float e = 1.0e-10;
			return vec3(abs(q.z + (q.w - q.y) / (6.0 * d + e)), d / (q.x + e), q.x);
		}

		vec3 hsv2rgb(vec3 c)
		{
			vec4 K = vec4(1.0, 2.0 / 3.0, 1.0 / 3.0, 3.0);
			vec3 p = abs(fract(c.xxx + K.xyz) * 6.0 - K.www);
			return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y);
		}

		void main(void) {
			vec4 baseRGB = texture2D(uSampler, vTextureCoord);
			vec4 overlayRGB = texture2D(overlay, vTextureCoord);
			
			if(overlayRGB.a > 0.0) {
				vec3 baseHSV = rgb2hsv(baseRGB.rgb);
				vec3 overlayHSV = rgb2hsv(overlayRGB.rgb);

				// Hue
				// vec3 resultHSV = vec3(overlayHSV.x, baseHSV.y, baseHSV.z);

				// Saturation
				// vec3 resultHSV = vec3(baseHSV.x, overlayHSV.y, baseHSV.z);

				// Value
				vec3 resultHSV = vec3(baseHSV.x, baseHSV.y, overlayHSV.z);

				vec3 resultRGB = hsv2rgb(resultHSV);
			
				gl_FragColor = vec4(resultRGB.rgb, baseRGB.a);
			}
			else {
				gl_FragColor = baseRGB;
			}
		}
		
	</script>
	
	<script type="text/javascript" src="replaceTest.js"></script>
	
</body>
</html>

And here's the JS:

var width = window.innerWidth;
var height = window.innerHeight;
var renderer = new PIXI.WebGLRenderer(width, height);
document.body.appendChild(renderer.view);

var stage = new PIXI.Container();

var sprite = PIXI.Sprite.fromImage('flat.png');
sprite.x = width / 2;//Set it at the center of the screen
sprite.y = height / 2;
sprite.anchor.set(0.5);//Make sure the center point of the image is at its center, instead of the default top left
stage.addChild(sprite);

//Create a uniforms object to send to the shader
var uniforms = {}

uniforms.overlay = {
	type:'sampler2D',
	value: PIXI.Texture.fromImage('stripesVal.png') // or stripesSat, stripesHue, etc
}

//Get shader code as a string
var shaderCode = document.getElementById("shader").innerHTML;

//Create our Pixi filter using our custom shader code
var rasShader = new PIXI.Filter(null,shaderCode,uniforms);
console.log(rasShader.uniforms);
sprite.filters = [rasShader];


function update() {
    requestAnimationFrame(update);
    renderer.render(stage);
}
update();

Any help would be appreciated!

ivan.popelyshev · November 22, 2017

myTexture.baseTexture.premultiplipliedAlpha=false;

Try that for textures that you use inside.

https://github.com/pixijs/pixi.js/blob/dev/src/core/textures/BaseTexture.js#L148

Also, uSampler will be premultiplied too, so either make sure it does have alpha=1.0 under your filter, either do something like "rgb = baseRgb.rgb / baseRgb.a;" , after you check their "a" component for zero.

https://github.com/pixijs/pixi-picture/blob/master/src/OverlayShader.ts#L3 - you see that i have to divide by alpha in some places because all textures are premultiplied, and the result is premultiplied too, so make sure you multiply gl_FragColor by alpha.

The problem is that premultiplied textures store (Ra,GaBa,a) instead of just (R,G,B,a). We use it for better linear filtering. Either you somehow change texture format before its uploaded to videomemory, either you divide it in shader. Also, uSampler and result is always premultiplied.

Blackdrazon · November 23, 2017

Thanks for that, that's solved the borders and the "teeth." Unfortunately, the result image still doesn't match the overlay, i.e. the stripes are still oversized and the fifth stripes has been pushed off the bottom. Is there anything else you can spot, or did I maybe just misunderstand part of your instructions?

Revised shader code:

		void main(void) {
			vec4 baseRGBPremul = texture2D(uSampler, vTextureCoord);
			vec4 overlayRGBPremul = texture2D(overlay, vTextureCoord);
			
			if(overlayRGBPremul.a > 0.0) {
				vec3 baseRGB = baseRGBPremul.rgb / baseRGBPremul.a;
				vec3 baseHSV = rgb2hsv(baseRGB);
				vec3 overlayRGB = overlayRGBPremul.rgb / overlayRGBPremul.a;
				vec3 overlayHSV = rgb2hsv(overlayRGB);
				
				// Hue
				// vec3 resultHSV = vec3(overlayHSV.x, baseHSV.y, baseHSV.z);

				// Saturation
				// vec3 resultHSV = vec3(baseHSV.x, overlayHSV.y, baseHSV.z);

				// Value
				vec3 resultHSV = vec3(baseHSV.x, baseHSV.y, overlayHSV.z);
				
				vec3 resultRGB = hsv2rgb(resultHSV);
			
				gl_FragColor = vec4(resultRGB * baseRGBPremul.a, baseRGBPremul.a);
			}
			else {
				gl_FragColor = baseRGBPremul;
			}
		}

Obviously, I'm not using premultipliedAlpha in this version.

EDIT: Not sure if this is helpful, but I just discovered that if the images are larger, the effect is even more exaggerated. Here's what happens if both the base and overlay are at 128x128:

Screenshot_14.png.0b9a6f3adeaf38bc31df850ccb42de66.png

ivan.popelyshev · November 23, 2017

You haven't map textureCoord. https://github.com/pixijs/pixi.js/wiki/v4-Creating-Filters - v4 ues temporary pow2 render textures, and you have to use " * filterArea.xy / dimensions" trick.

Look at https://github.com/pixijs/pixi-filters/tree/master/filters/simple-lightmap/src as an example.

I'm very sorry that filters and shaders are that difficult in v4. We are fixing it for v5.

And one more thing: after you fix filters, try to make renderer plugin for a sprite. Look at how https://github.com/pixijs/pixi-picture implements OVERLAY. It takes image directly under the sprite and blends it.

Blackdrazon · November 24, 2017

We're definitely closer to the mark now, but I'm afraid it's still slightly off:

Screenshot_16.png.a7125e1535a93abf315dffba5b7e37f4.png stripesVal2.png.3c06bf6b743d78fcd55bbb13e61a69a7.png

On the left you see the current output for the 128x128 sprite, and on the right, my mockup showing where the stripes should be (I just took the 128x128 overlay and added the red and the black border, so it should be accurate). As you can see, the stripes are still off their marks and too thick in the final image, unfortunately.

Current code is below:

		varying vec2 vTextureCoord;
		uniform sampler2D uSampler;
		uniform sampler2D overlay;
		uniform vec4 filterArea;
		uniform vec2 dimensions;

                [...]

		void main(void) {
			vec4 baseRGBPremul = texture2D(uSampler, vTextureCoord);
			
			vec2 overlayCoord = (vTextureCoord * filterArea.xy) / dimensions;		
			vec4 overlayRGBPremul = texture2D(overlay, overlayCoord);
			
			// A transparent tile on the base doesn't need to be touched, and a transparent tile on
			// the overlay means "no change."
			if(baseRGBPremul.a > 0.0 && overlayRGBPremul.a > 0.0) {
				vec3 baseRGB = baseRGBPremul.rgb / baseRGBPremul.a;
				vec3 baseHSV = rgb2hsv(baseRGB);
				vec3 overlayRGB = overlayRGBPremul.rgb / overlayRGBPremul.a;
				vec3 overlayHSV = rgb2hsv(overlayRGB);
				
				//vec3 resultHSV = vec3(overlayHSV.x, baseHSV.y, baseHSV.z);
				//vec3 resultHSV = vec3(baseHSV.x, overlayHSV.y, baseHSV.z);
				vec3 resultHSV = vec3(baseHSV.x, baseHSV.y, overlayHSV.z);
				
				vec3 resultRGB = hsv2rgb(resultHSV);
			
				gl_FragColor = vec4(resultRGB * baseRGBPremul.a, baseRGBPremul.a);
			}
			else {
				gl_FragColor = baseRGBPremul;
			}
		}

Javascript:

var width = window.innerWidth;
var height = window.innerHeight;
var renderer = new PIXI.WebGLRenderer(width, height);
//renderer.backgroundColor = 0xFFFFFF;
document.body.appendChild(renderer.view);

//The stage is the root container that will hold everything in our scene
var stage = new PIXI.Container();

//Load an image and create an object
var sprite = PIXI.Sprite.fromImage('flat.png');
sprite.x = width / 2;
sprite.y = height / 2;

//Make sure the center point of the image is at its center, instead of the default top left
sprite.anchor.set(0.5);

stage.addChild(sprite);


//Create a uniforms object to send to the shader
var uniforms = {}
var overlayTex = PIXI.Texture.fromImage('stripesVal.png');

uniforms.overlay = {
	type:'sampler2D',
	value: overlayTex
}
uniforms.dimensions = {
	type:'vec2',
	value: []
};


//Get shader code as a string
var shaderCode = document.getElementById("shader").innerHTML;

//Create our Pixi filter using our custom shader code


var rasShader = new PIXI.Filter(null,shaderCode,uniforms);

rasShader.autoFit = false;

rasShader.apply = function(filterManager, input, output) {
	this.uniforms.dimensions[0] = input.sourceFrame.width;
	this.uniforms.dimensions[1] = input.sourceFrame.height;
	
	filterManager.applyFilter(this, input, output);
}

sprite.filters = [rasShader];

Thanks for all the help again, and I will check out that renderer plugin link once this is working!

ivan.popelyshev · November 24, 2017

According to https://github.com/pixijs/pixi.js/blob/dev/src/core/renderers/webgl/filters/Filter.js#L78 , filters have padding 4 by default. Set it to 0.

Blackdrazon · November 25, 2017

That's got it! And it also retroactively explains the "teeth" to me! Thank you very much!

Sign In

Pixi Filter/Shader with Confusing Results

Recommended Posts

Blackdrazon

Link to comment

Share on other sites

ivan.popelyshev

Link to comment

Share on other sites

Blackdrazon

Link to comment

Share on other sites

ivan.popelyshev

Link to comment

Share on other sites

Blackdrazon

Link to comment

Share on other sites

ivan.popelyshev

Link to comment

Share on other sites

Blackdrazon

Link to comment

Share on other sites

Join the conversation

Recently Browsing 0 members

Browse

Activity