PSX Inverse Hull Rim Light Shader

Shader "psx/RimClipStencil"
{
    Properties
    {
        _MainTex ("Texture", 2D) = "white" {}
        _Color ("Base Color", Color) = (1.0, 1.0, 1.0, 1.0)
        _Light ("Base Light", Color) = (0.0, 0.0, 0.0, 1.0)
        _LightAmp("Light Amplification", Range(0,10)) = 1.0
        _SnapSize ("Snap Size", Float) = 1.0
        _FogStrength("Fog Strength", Range(0,1)) = 1.0

        // Outline settings
        _OutlineSize("Outline Size", Range(0,0.1)) = 0.006666667
        _OutlineOffset("Outline Offset", Range(0,0.1)) = 0.006666667
        _OutlineIntensity("Outline Intensity", Range(0.0,10.0)) = 1.0
        _OutlineBlend("Outline Texture Blend", Range(0.0,1.0)) = 0.8
        [IntRange] _OutlineStencilIndex("Outline Stencil ID", Range(0,255)) = 0

        //Shader feature toggles
        //[Toggle(OUTLINE_SHARED_OFFSET)] _OutlineSharedOffset ("Outline Offset from Shared Origin", Float) = 0.0
        [Toggle(OUTLINE_FROM_NORMALS)] _OutlineFromNormals ("Outline Normals use Surface Normals", Float) = 0.0
    }
    SubShader
    {
        Tags { "RenderType"="Opaque" "LightMode"="Vertex" }
        LOD 100

        Pass
        {
            Name "BasePass"
            Cull Back

            // This stencil buffer writes the given Stencil ID to all pixels occupied by the model, so meshes with the given index share a contiguous outline
            Stencil
            {
                Ref [_OutlineStencilIndex]
                Comp Always
                Pass Replace
            }

            CGPROGRAM
            #pragma vertex vert
            #pragma fragment frag

            #include "UnityCG.cginc"

            // Static configurable values
            static const int MAX_LIGHTS = 8; // How many lights affect objects. RANGE: 0 to 8
            static const half LIGHT_BIAS = 0.15; // Sharpening factor for attenuation. RANGE: 0.0 to 0.5

            int _OutlineStencilIndex;

            sampler2D _MainTex;
            float4 _MainTex_ST;

            float4 _Color;
            float4 _Light;
            float _LightAmp;
            float _SnapSize;

            float _FogStrength;
            uniform half4 unity_FogStart;
            uniform half4 unity_FogEnd;

            struct appdata
            {
                float4 vertex : POSITION;
                float4 normal : NORMAL;
                fixed4 vertex_color : COLOR;
                float2 uv : TEXCOORD0;
            };

            struct v2f
            {
                float4 vertex : SV_POSITION;
                float4 color : COLOR0;
                float4 fog_color : COLOR1;
                noperspective float2 uv : TEXCOORD0;
            };

            v2f vert (appdata v)
            {
                v2f o;

                float4 clip_pos = UnityObjectToClipPos (v.vertex);
                float3 view_pos = UnityObjectToViewPos (v.vertex.xyz);
                float view_distance = length( mul(UNITY_MATRIX_MV, v.vertex) );

                // Calculate vertex snapping in clip space terms
                float snap_factor = 1 / _SnapSize;
                float4 snapped_pos = clip_pos;
                snapped_pos.xyz = clip_pos.xyz / clip_pos.w;
                snapped_pos.x = floor(160 * snap_factor * snapped_pos.x) / (160 * snap_factor);
                snapped_pos.y = floor(120 * snap_factor * snapped_pos.y) / (120 * snap_factor);
                snapped_pos.xyz *= clip_pos.w;
                o.vertex = snapped_pos;

                // Wrapped vertex lighting, brightest to darkest, which ignores normals
                float3 light_color = unity_AmbientSky.rgb;

                for(int light_id = 0; light_id < MAX_LIGHTS; light_id++)
                {
                    // Calculate directional light vector, or vector to point light source
                    float3 to_light = unity_LightPosition[light_id].xyz - view_pos.xyz * unity_LightPosition[light_id].w;
                    float length_sq = dot (to_light, to_light);

                    // Skip this light if not directional and outside range
                    if (unity_LightPosition[light_id].w && length_sq > unity_LightAtten[light_id].w)
                    {
                        continue;
                    }

                    // Prevents NaNs when overlapping light position
                    length_sq = max(length_sq, 0.00001);
                    to_light *= rsqrt(length_sq);

                    // Calculate and apply light attenuation, sharpened with LIGHT_BIAS
                    float attenuation = 1.0 / (1.0 + length_sq * unity_LightAtten[light_id].z);
                    float biased_atten = smoothstep(LIGHT_BIAS, 1.0 - LIGHT_BIAS, attenuation);
                    light_color += unity_LightColor[light_id].rgb * biased_atten * _LightAmp;
                }

                // Surface is lit to the maximum of base light or environmental light
                o.color = _Color;
                o.color.rgb *= max(light_color, _Light.rgb * _LightAmp);

                // Fog with vertex cutout
                float4 fog_color = unity_FogColor;
                float fog_density = (unity_FogEnd - view_distance) / (unity_FogEnd - unity_FogStart);
                o.fog_color = fog_color;
                o.fog_color.a = saturate(fog_density) * _FogStrength;

                o.uv = TRANSFORM_TEX(v.uv, _MainTex);
                return o;
            }

            fixed4 frag (v2f i) : SV_Target
            {
                // sample the texture, and modulate by light color
                fixed4 col = tex2D(_MainTex, i.uv);
                fixed4 final_col = col * i.color;

                // apply fog color
                float fog_blend = (_FogStrength - i.fog_color.a) * _FogStrength;
                final_col.rgb = lerp(final_col.rgb, i.fog_color.rgb, fog_blend);

                return final_col;
            }
            ENDCG
        }

        Pass
        {
            Name "OutlinePass"
            Cull Back

            Stencil
            {
                Ref [_OutlineStencilIndex]
                Comp Greater
            }

            CGPROGRAM
                #pragma shader_feature OUTLINE_FROM_NORMALS
                #pragma vertex vert
                #pragma fragment frag
                #include "UnityCG.cginc"

                static const int MAX_LIGHTS = 8;
                static const half OUTLINE_FALLOFF = 0.5;

                float4 _Color;
                float4 _Light;
                float _LightAmp;

                half _OutlineSize;
                half _OutlineOffset;
                half _OutlineIntensity;
                half _OutlineBlend;

                half4 _OutlineTestNormal;

                sampler2D _MainTex;
                float4 _MainTex_ST;

                float _SnapSize;

                float _FogStrength;
                uniform half4 unity_FogStart;
                uniform half4 unity_FogEnd;

                struct appdata
                {
                    float4 vertex : POSITION;
                    float3 normal : NORMAL;
                    half4 vertex_color : COLOR;
                    float2 uv : TEXCOORD0;
                };

                struct v2f
                {
                    // noperspective keyword prevents modern affine warping correction
                    float4 vertex : SV_POSITION;
                    half4 color : COLOR0;
                    float4 fog_color : COLOR1;
                    noperspective float2 uv : TEXCOORD0;
                };

                v2f vert (appdata v)
                {
                    v2f o;

                    float3 view_pos = UnityObjectToViewPos (v.vertex);
                    float4 clip_pos = UnityObjectToClipPos (v.vertex);
                    float view_distance = length( mul(UNITY_MATRIX_MV, v.vertex) );

                    half3 light_color = unity_AmbientSky.rgb;
                    float3 swizzled_normal = v.normal;

                    #if (!OUTLINE_FROM_NORMALS)
                    {
                        // Remap and swizzle Blender-style baked normals to Unity normals in range -1 to +1
                        swizzled_normal.rgb = normalize(v.vertex_color.gbr * 2.0 - 1.0);
                        swizzled_normal.r *= -1.0;
                        swizzled_normal.b *= -1.0;
                    }
                    #endif

                    // Calculate distance and direction based light factors, for offsetting rimlight
                    float3 avg_light_normal = float3(0.0, 0.0, 0.0);
                    float max_light_atten = 0.0;
                    float total_luminance = 0.0;
                    float valid_lights = 0;

                    // For each light, add vertex light intensity at object's origin
                    for(int light_id = 0; light_id < MAX_LIGHTS; light_id++)
                    {
                        float3 to_light = unity_LightPosition[light_id].xyz - view_pos.xyz * unity_LightPosition[light_id].w;
                        //float3 to_light = clip_light_pos.xyz - view_pos.xyz * clip_light_pos.w;
                        float length_sq = dot (to_light, to_light);

                        // Skip this light if not directional and outside range
                        if (unity_LightPosition[light_id].w && length_sq > unity_LightAtten[light_id].w)
                        {
                            continue;
                        }

                        // Prevents NaNs when overlapping light position
                        length_sq = max(length_sq, 0.00001);
                        to_light *= rsqrt(length_sq);

                        // Calculate and apply light attenuation, sharpened with OUTLINE_FALLOFF
                        float light_quad_atten = unity_LightAtten[light_id].z;

                        float attenuation = 1.0 / (1.0 + length_sq * unity_LightAtten[light_id].z);
                        float biased_atten = smoothstep(0.0, OUTLINE_FALLOFF, attenuation);

                        valid_lights += 1;
                        max_light_atten = max(max_light_atten, attenuation);
                        avg_light_normal += to_light * attenuation;

                        light_color += unity_LightColor[light_id].rgb * biased_atten * _LightAmp * _OutlineIntensity;
                        total_luminance += Luminance(unity_LightColor[light_id].rgb * attenuation);
                    }

                    total_luminance = saturate(total_luminance);

                    float3 outline_normal = UnityObjectToWorldNormal(swizzled_normal);
                    float3 clip_normal = mul((float3x3) UNITY_MATRIX_VP, mul((float3x3) UNITY_MATRIX_M, swizzled_normal));

                    float clip_aspect = _ScreenParams.x / _ScreenParams.y;
                    float2 clip_scale_offset = normalize(clip_normal.xy) * total_luminance * _OutlineSize * clip_pos.w;
                    clip_scale_offset.y *= clip_aspect;
                    clip_pos.xy += clip_scale_offset;

                    //avg_light_normal = _OutlineTestNormal;
                    float3 clip_light_normal = mul((float3x3) UNITY_MATRIX_P, avg_light_normal);
                    //float3 clip_light_normal = mul((float3x3) UNITY_MATRIX_VP, mul((float3x3) UNITY_MATRIX_M, avg_light_normal));

                    float2 clip_light_offset = normalize(clip_light_normal.xy) * total_luminance * _OutlineOffset * clip_pos.w;
                    clip_light_offset.y *= clip_aspect;
                    clip_pos.xy += clip_light_offset;

                    avg_light_normal = normalize(avg_light_normal);
                    avg_light_normal = mul(avg_light_normal, (float3x3)UNITY_MATRIX_IT_MV);

                    float3 scale_offset = normalize(outline_normal) * max_light_atten * _OutlineSize; // - normalize(outline_normal) * 0.01;
                    float3 light_offset = normalize(avg_light_normal) * max_light_atten * _OutlineOffset;
                    float3 offset_pos = scale_offset + light_offset;

                    v.vertex.xyz += offset_pos;

                    // Calculate vertex snapping in clip space terms
                    float snap_factor = 1 / _SnapSize;
                    float4 snapped_pos = clip_pos;
                    snapped_pos.xyz = clip_pos.xyz / clip_pos.w;
                    snapped_pos.x = floor(160 * snap_factor * snapped_pos.x) / (160 * snap_factor);
                    snapped_pos.y = floor(120 * snap_factor * snapped_pos.y) / (120 * snap_factor);
                    snapped_pos.xyz *= clip_pos.w;

                    o.vertex = snapped_pos;

                    // Surface is lit to the maximum of base light or environmental light
                    o.color = _Color;
                    o.color.rgb *= max(light_color, _Light.rgb * _LightAmp * _OutlineIntensity);

                    // Fog with vertex cutout
                    float4 fog_color = unity_FogColor;
                    float fog_density = (unity_FogEnd - view_distance) / (unity_FogEnd - unity_FogStart);
                    o.fog_color = fog_color;
                    o.fog_color.a = saturate(fog_density) * _FogStrength;

                    o.uv = TRANSFORM_TEX(v.uv, _MainTex);
                    return o;
                }

                fixed4 frag (v2f i) : SV_Target
                {
                    // sample the texture
                    fixed4 col = tex2D(_MainTex, i.uv);

                    // Interpolate pure light color with recolored texture
                    fixed4 final_col = lerp(i.color, col * i.color, _OutlineBlend);

                    // apply fog color
                    float fog_blend = (_FogStrength - i.fog_color.a) * _FogStrength;
                    final_col.rgb = lerp(final_col.rgb, i.fog_color.rgb, fog_blend);

                    return final_col;
                }
            ENDCG
        }
    }
}