rdi_platform_windows/
shader.rs

1//! Runtime HLSL compilation for the Windows sprite renderer.
2//!
3//! WARNING: This proof of concept (PoC) is not shader-injection-proof.
4//! Only use shaders from sources you trust. Use untrusted shaders at your own risk.
5
6pub use crate::shader_library::{BuiltinShader, GLITCH, IDENTITY, PARTICLE_VORTEX};
7use crate::shader_library::{
8    DEFAULT_PIXEL, DEFAULT_VERTEX, HEADER, PIXEL_DECLARATION, VERTEX_DECLARATION, VERTEX_FOOTER,
9    pipeline_template,
10};
11use rdi_core::{DesktopError, ShaderPipeline, ShaderProgram};
12use windows::Win32::Graphics::Direct3D::Fxc::{
13    D3DCOMPILE_ENABLE_STRICTNESS, D3DCOMPILE_OPTIMIZATION_LEVEL3, D3DCompile,
14};
15use windows::Win32::Graphics::Direct3D::ID3DBlob;
16use windows::core::{PCSTR, s};
17
18#[derive(Clone, Debug, Default)]
19pub struct PassSource {
20    pub vertex: Option<String>,
21    pub pixel: Option<String>,
22    pub draw: rdi_core::DrawSpec,
23    pub inputs: Vec<usize>,
24    pub output: Option<usize>,
25    pub blend: rdi_core::PassBlend,
26}
27
28#[derive(Clone, Debug, Default)]
29pub struct ExecutionSource {
30    pub parameters: Vec<rdi_core::EffectParameter>,
31    pub targets: Vec<rdi_core::EffectTarget>,
32    pub passes: Vec<PassSource>,
33    pub body_artwork: bool,
34    pub label_artwork: bool,
35}
36
37#[derive(Clone, Debug)]
38pub struct ShaderSource<'a> {
39    pub execution: Option<std::sync::Arc<ExecutionSource>>,
40    pub pipeline: ShaderPipeline,
41    pub pixel: Option<&'a str>,
42    pub vertex: Option<&'a str>,
43}
44
45impl<'a> From<&'a str> for ShaderSource<'a> {
46    fn from(pixel: &'a str) -> Self {
47        Self {
48            execution: None,
49            pipeline: ShaderPipeline::Sprite,
50            pixel: Some(pixel),
51            vertex: None,
52        }
53    }
54}
55
56impl<'a> From<&'a String> for ShaderSource<'a> {
57    fn from(pixel: &'a String) -> Self {
58        pixel.as_str().into()
59    }
60}
61
62/// Compile a complete trusted shader program from a descriptor or pixel-only HLSL.
63/// No filesystem includes or GPU resources are accessed by this operation.
64pub fn compile<'a>(source: impl Into<ShaderSource<'a>>) -> Result<ShaderProgram, DesktopError> {
65    let source = source.into();
66    if source.execution.is_some() && source.pipeline != ShaderPipeline::Procedural {
67        return Err(DesktopError::InvalidEffect("execution requires procedural pipeline".into()));
68    }
69    let mut program = compile_stages(&source)?;
70    if source.pipeline == ShaderPipeline::Procedural {
71        let recipe = source.execution.as_deref().cloned().unwrap_or_else(|| ExecutionSource {
72            passes: vec![PassSource::default()], ..Default::default()
73        });
74        if recipe.passes.len() > 8 || recipe.parameters.len() > 16 || recipe.targets.len() > 4 {
75            return Err(DesktopError::InvalidEffect("procedural descriptor exceeds limits".into()));
76        }
77        let source_bytes: usize = recipe.passes.iter().map(|pass| pass.vertex.as_ref().map_or(0, String::len) + pass.pixel.as_ref().map_or(0, String::len)).sum();
78        if source_bytes > 1024 * 1024 { return Err(DesktopError::InvalidEffect("pass sources exceed 1 MiB".into())); }
79        let mut passes = Vec::new();
80        for pass in recipe.passes {
81            let stages = if pass.vertex.is_none() && pass.pixel.is_none() { program.clone() } else {
82                compile_stages(&ShaderSource { execution: None, pipeline: ShaderPipeline::Procedural,
83                    vertex: pass.vertex.as_deref().or(source.vertex), pixel: pass.pixel.as_deref().or(source.pixel) })?
84            };
85            passes.push(rdi_core::EffectPass { vertex_bytecode: stages.vertex_bytecode, pixel_bytecode: stages.pixel_bytecode,
86                draw: pass.draw, inputs: pass.inputs, output: pass.output, blend: pass.blend });
87        }
88        let execution = rdi_core::EffectExecution { parameters: recipe.parameters, targets: recipe.targets, passes,
89            body_artwork: recipe.body_artwork, label_artwork: recipe.label_artwork };
90        execution.validate(&execution.defaults())?;
91        program.execution = Some(std::sync::Arc::new(execution));
92    }
93    Ok(program)
94}
95
96fn compile_stages(source: &ShaderSource<'_>) -> Result<ShaderProgram, DesktopError> {
97    if source
98        .pixel
99        .map_or(0, str::len)
100        .saturating_add(source.vertex.map_or(0, str::len))
101        > 1024 * 1024
102    {
103        return Err(DesktopError::InvalidEffect(
104            "shader source exceeds 1 MiB".into(),
105        ));
106    }
107    let (header, footer, pixel_entry) = pipeline_template(source.pipeline);
108    let vertex_source = source.vertex.unwrap_or(DEFAULT_VERTEX);
109    let pixel_source = source.pixel.unwrap_or(DEFAULT_PIXEL);
110    let vertex = format!(
111        "{HEADER}{header}\n{VERTEX_DECLARATION}\n#line 1 \"vertex.hlsl\"\n{vertex_source}\n#line 1 \"rdi_wrapper.hlsl\"\n{VERTEX_FOOTER}"
112    );
113    let pixel = format!(
114        "{HEADER}{header}\n{PIXEL_DECLARATION}\n#line 1 \"pixel.hlsl\"\n{pixel_source}\n#line 1 \"rdi_wrapper.hlsl\"\n{footer}"
115    );
116    Ok(ShaderProgram {
117        execution: None,
118        vertex_bytecode: compile_stage(&vertex, true, s!("rdi_vertex"))?.into(),
119        pixel_bytecode: compile_stage(&pixel, false, pixel_entry)?.into(),
120        pipeline: source.pipeline,
121    })
122}
123
124fn compile_stage(source: &str, vertex: bool, entry: PCSTR) -> Result<Vec<u8>, DesktopError> {
125    let mut code: Option<ID3DBlob> = None;
126    let mut errors: Option<ID3DBlob> = None;
127    // SAFETY: source bytes outlive compilation; output slots are valid; no include handler is installed.
128    let result = unsafe {
129        D3DCompile(
130            source.as_ptr().cast(),
131            source.len(),
132            s!("rdi.hlsl"),
133            None,
134            None,
135            entry,
136            if vertex { s!("vs_5_0") } else { s!("ps_5_0") },
137            D3DCOMPILE_ENABLE_STRICTNESS | D3DCOMPILE_OPTIMIZATION_LEVEL3,
138            0,
139            &mut code,
140            Some(&mut errors),
141        )
142    };
143    if let Err(error) = result {
144        let message = errors
145            .as_ref()
146            .map(|blob| String::from_utf8_lossy(&blob_bytes(blob)).into_owned())
147            .unwrap_or_else(|| error.to_string());
148        return Err(DesktopError::InvalidEffect(message));
149    }
150    code.as_ref()
151        .map(blob_bytes)
152        .ok_or_else(|| DesktopError::InvalidEffect("compiler returned no bytecode".into()))
153}
154
155fn blob_bytes(blob: &ID3DBlob) -> Vec<u8> {
156    // SAFETY: the blob owns GetBufferSize initialized bytes until after the copy.
157    unsafe {
158        std::slice::from_raw_parts(blob.GetBufferPointer().cast::<u8>(), blob.GetBufferSize())
159            .to_vec()
160    }
161}
162
163#[cfg(test)]
164mod tests {
165    use super::*;
166
167    #[test]
168    fn builtin_catalog_compiles_and_preserves_aliases() {
169        let mut names = std::collections::BTreeSet::new();
170        for &builtin in BuiltinShader::ALL {
171            assert!(names.insert(builtin.name()), "duplicate built-in name");
172            assert_eq!(builtin.name().parse::<BuiltinShader>().unwrap(), builtin);
173            let program = compile(builtin).unwrap();
174            assert_eq!(program, compile(builtin.source()).unwrap());
175            assert_eq!(program.pipeline, builtin.source().pipeline);
176            assert!(!program.vertex_bytecode.is_empty());
177            assert!(!program.pixel_bytecode.is_empty());
178        }
179        for (builtin, alias) in [
180            (BuiltinShader::Identity, compile(IDENTITY).unwrap()),
181            (BuiltinShader::Glitch, compile(GLITCH).unwrap()),
182            (
183                BuiltinShader::ParticleVortex,
184                compile(PARTICLE_VORTEX).unwrap(),
185            ),
186        ] {
187            assert_eq!(compile(builtin).unwrap(), alias);
188        }
189        assert!("unknown".parse::<BuiltinShader>().is_err());
190    }
191
192    #[test]
193    fn rejects_incompatible_stage_entries() {
194        for pipeline in [ShaderPipeline::Sprite, ShaderPipeline::Particles, ShaderPipeline::Procedural] {
195            for vertex in [
196                "float4 vertex(Instance instance, uint vertex_id : SV_VertexID) : SV_Position { return default_vertex(instance, vertex_id).position; }",
197                "VertexOutput vertex(uint vertex_id : SV_VertexID) { return (VertexOutput)0; }",
198            ] {
199                assert!(
200                    compile(ShaderSource {
201                        execution: None,
202                        pipeline,
203                        vertex: Some(vertex),
204                        pixel: None
205                    })
206                    .is_err()
207                );
208            }
209            for pixel in [
210                "float pixel(VertexOutput input) : SV_Target { return 1; }",
211                "float4 pixel(float2 uv : TEXCOORD0) : SV_Target { return float4(uv, 0, 1); }",
212            ] {
213                assert!(
214                    compile(ShaderSource {
215                        execution: None,
216                        pipeline,
217                        vertex: None,
218                        pixel: Some(pixel)
219                    })
220                    .is_err()
221                );
222            }
223        }
224    }
225
226    #[test]
227    fn compiles_runtime_programs_and_reports_source_errors() {
228        assert!(compile(IDENTITY).is_ok());
229        assert!(compile(GLITCH).is_ok());
230        let particles = compile(PARTICLE_VORTEX).unwrap();
231        assert!(!particles.vertex_bytecode.is_empty());
232        assert_eq!(
233            compile(GLITCH).unwrap(),
234            compile(GLITCH.pixel.unwrap()).unwrap()
235        );
236        for pipeline in [ShaderPipeline::Sprite, ShaderPipeline::Particles, ShaderPipeline::Procedural] {
237            for vertex in [None, Some(DEFAULT_VERTEX)] {
238                for pixel in [None, Some(DEFAULT_PIXEL)] {
239                    let program = compile(ShaderSource {
240                        execution: None,
241                        pipeline,
242                        vertex,
243                        pixel,
244                    })
245                    .unwrap();
246                    assert_eq!(program.pipeline, pipeline);
247                    assert!(!program.vertex_bytecode.is_empty());
248                    assert!(!program.pixel_bytecode.is_empty());
249                }
250            }
251        }
252        assert!(
253            compile(ShaderSource {
254                execution: None,
255                vertex: Some("invalid vertex"),
256                ..PARTICLE_VORTEX
257            })
258            .unwrap_err()
259            .to_string()
260            .contains("vertex.hlsl")
261        );
262        assert!(
263            compile(ShaderSource {
264                execution: None,
265                pixel: Some("invalid pixel"),
266                ..PARTICLE_VORTEX
267            })
268            .unwrap_err()
269            .to_string()
270            .contains("pixel.hlsl")
271        );
272        assert!(
273            compile(ShaderSource {
274                execution: None,
275                vertex: Some(&" ".repeat(1024 * 1024)),
276                ..PARTICLE_VORTEX
277            })
278            .is_err()
279        );
280        assert!(
281            compile("not valid hlsl")
282                .unwrap_err()
283                .to_string()
284                .contains("pixel.hlsl")
285        );
286        assert!(compile("#include \"external.hlsl\"").is_err());
287    }
288}
rdi_platform_windows/shader.rs

rdi_platform_windows/
shader.rs
