Make layer checkout functions return Option<Layer> (#69)

Even if the functions succeed, the returned layer pointer is sometimes null.
virtualritz · Oct 31, 2024 · 76660c1 · 76660c1
1 parent 74bcf4c
commit 76660c1
Show file tree

Hide file tree

Showing 7 changed files with 49 additions and 32 deletions.
diff --git a/after-effects/src/pf/render.rs b/after-effects/src/pf/render.rs
@@ -331,24 +331,27 @@ impl SmartRenderCallbacks {
         self.rc_ptr
     }
 
-    pub fn checkout_layer_pixels(&self, checkout_id: u32) -> Result<Layer, Error> {
+    /// Check out a layer corresponding to the ID you checked it out with in SmartPreRender.
+    /// Note that the resulting layer may be None even if the ID is valid--for example, if the target layer is an
+    /// adjustment layer with nothing beneath it.
+    pub fn checkout_layer_pixels(&self, checkout_id: u32) -> Result<Option<Layer>, Error> {
         if let Some(checkout_layer_pixels) = unsafe { *self.rc_ptr }.checkout_layer_pixels {
-            let mut effect_world_ptr =
-                std::mem::MaybeUninit::<*mut ae_sys::PF_EffectWorld>::uninit();
+            let mut effect_world_ptr = std::ptr::null_mut();
 
-            match unsafe {
+            match (unsafe {
                 checkout_layer_pixels(
                     (*self.in_data_ptr).effect_ref,
                     checkout_id as i32,
-                    effect_world_ptr.as_mut_ptr(),
+                    &mut effect_world_ptr,
                 )
-            } {
-                0 => Ok(Layer::from_raw(
-                    unsafe { effect_world_ptr.assume_init() },
+            }, effect_world_ptr.is_null()) {
+                (0, false) => Ok(Some(Layer::from_raw(
+                    effect_world_ptr,
                     self.in_data_ptr,
                     None,
-                )),
-                e => Err(Error::from(e)),
+                ))),
+                (0, true) => Ok(None),
+                (e, _) => Err(Error::from(e)),
             }
         } else {
             Err(Error::InvalidCallback)
@@ -368,23 +371,25 @@ impl SmartRenderCallbacks {
         }
     }
 
-    pub fn checkout_output(&self) -> Result<Layer, Error> {
+    /// Check out the output layer. Note that the resulting layer may be None in certain cases--for example, if the
+    /// target layer is an adjustment layer with nothing beneath it.
+    pub fn checkout_output(&self) -> Result<Option<Layer>, Error> {
         if let Some(checkout_output) = unsafe { *self.rc_ptr }.checkout_output {
-            let mut effect_world_ptr =
-                std::mem::MaybeUninit::<*mut ae_sys::PF_EffectWorld>::uninit();
+            let mut effect_world_ptr = std::ptr::null_mut();
 
-            match unsafe {
+            match (unsafe {
                 checkout_output(
                     (*self.in_data_ptr).effect_ref,
-                    effect_world_ptr.as_mut_ptr(),
+                    &mut effect_world_ptr,
                 )
-            } {
-                0 => Ok(Layer::from_raw(
-                    unsafe { effect_world_ptr.assume_init() },
+            }, effect_world_ptr.is_null()) {
+                (0, false) => Ok(Some(Layer::from_raw(
+                    effect_world_ptr,
                     self.in_data_ptr,
                     None,
-                )),
-                e => Err(Error::from(e)),
+                ))),
+                (0, true) => Ok(None),
+                (e, _) => Err(Error::from(e)),
             }
         } else {
             Err(Error::InvalidCallback)

diff --git a/examples/color_grid/src/lib.rs b/examples/color_grid/src/lib.rs
@@ -197,7 +197,9 @@ impl AdobePluginGlobal for Plugin {
                 let mut current_color = 0;
 
                 let cb = extra.callbacks();
-                let input_world = cb.checkout_layer_pixels(0)?;
+                let Some(input_world) = cb.checkout_layer_pixels(0)? else {
+                    return Ok(());
+                };
 
                 let param = params.get(Params::GridUI)?;
                 let colors = param.as_arbitrary()?.value::<ArbData>()?;
@@ -225,7 +227,7 @@ impl AdobePluginGlobal for Plugin {
                     let color16_g = (color32.green * ae::MAX_CHANNEL16 as f32) as u32;
                     let color16_b = (color32.blue  * ae::MAX_CHANNEL16 as f32) as u32;
 
-                    if let Ok(mut output_world) = cb.checkout_output() {
+                    if let Ok(Some(mut output_world)) = cb.checkout_output() {
                         let progress_final = output_world.height() as _;
 
                         origin = in_data.output_origin();

diff --git a/examples/histo_grid/src/lib.rs b/examples/histo_grid/src/lib.rs
@@ -266,7 +266,9 @@ impl AdobePluginInstance for Instance {
                 let mut current_color = 0;
 
                 let cb = extra.callbacks();
-                let input_world = cb.checkout_layer_pixels(0)?;
+                let Some(input_world) = cb.checkout_layer_pixels(0)? else {
+                    return Ok(());
+                };
 
                 let mut color_cache = ColorCache::new();
                 color_cache.compute_color_grid_from_frame(&input_world)?;
@@ -294,7 +296,7 @@ impl AdobePluginInstance for Instance {
                     let color16_g = (color32.green * ae::MAX_CHANNEL16 as f32) as u32;
                     let color16_b = (color32.blue  * ae::MAX_CHANNEL16 as f32) as u32;
 
-                    if let Ok(mut output_world) = cb.checkout_output() {
+                    if let Ok(Some(mut output_world)) = cb.checkout_output() {
                         let progress_final = output_world.height() as _;
 
                         origin = in_data.output_origin();

diff --git a/examples/resizer/src/lib.rs b/examples/resizer/src/lib.rs
@@ -249,9 +249,11 @@ impl AdobePluginGlobal for Plugin {
             }
             ae::Command::SmartRender { extra } => {
                 let cb = extra.callbacks();
-                let in_layer = cb.checkout_layer_pixels(0)?;
+                let Some(in_layer) = cb.checkout_layer_pixels(0)? else {
+                    return Ok(());
+                };
 
-                if let Ok(mut out_layer) = cb.checkout_output() {
+                if let Ok(Some(mut out_layer)) = cb.checkout_output() {
                     let border = params.get(Params::Amount)?.as_slider()?.value() as f32;
 
                     let color = params.get(Params::Color)?.as_color()?.value();

diff --git a/examples/rust_gpu/src/lib.rs b/examples/rust_gpu/src/lib.rs
@@ -111,9 +111,11 @@ impl AdobePluginGlobal for Plugin {
             }
             ae::Command::SmartRender { extra } => {
                 let cb = extra.callbacks();
-                let in_layer = cb.checkout_layer_pixels(0)?;
+                let Some(in_layer) = cb.checkout_layer_pixels(0)? else {
+                    return Ok(());
+                };
 
-                if let Ok(mut out_layer) = cb.checkout_output() {
+                if let Ok(Some(mut out_layer)) = cb.checkout_output() {
                     let in_size  = (in_layer.width()  as usize, in_layer.height()  as usize, in_layer.buffer_stride());
                     let out_size = (out_layer.width() as usize, out_layer.height() as usize, out_layer.buffer_stride());
 

diff --git a/examples/sdk_noise/src/lib.rs b/examples/sdk_noise/src/lib.rs
@@ -156,9 +156,11 @@ impl AdobePluginGlobal for Plugin {
             }
             ae::Command::SmartRender { extra } => {
                 let cb = extra.callbacks();
-                let input_world = cb.checkout_layer_pixels(0)?;
+                let Some(input_world) = cb.checkout_layer_pixels(0)? else {
+                    return Ok(());
+                };
 
-                if let Ok(mut output_world) = cb.checkout_output() {
+                if let Ok(Some(mut output_world)) = cb.checkout_output() {
                     let progress_final = output_world.height() as _;
 
                     let value = params.get(Params::Noise)?.as_float_slider()?.value() as f32 / 100.0;

diff --git a/examples/supervisor/src/lib.rs b/examples/supervisor/src/lib.rs
@@ -226,7 +226,9 @@ impl AdobePluginInstance for Instance {
             }
             ae::Command::SmartRender { mut extra } => {
                 let cb = extra.callbacks();
-                let input_world = cb.checkout_layer_pixels(0)?;
+                let Some(input_world) = cb.checkout_layer_pixels(0)? else {
+                    return Ok(());
+                };
 
                 let pre_render_data = extra.pre_render_data_mut::<PreRenderData>().unwrap();
                 if self.advanced_mode {
@@ -246,7 +248,7 @@ impl AdobePluginInstance for Instance {
 
                 let pixel_float = pre_render_data.color;
 
-                if let Ok(mut output_world) = cb.checkout_output() {
+                if let Ok(Some(mut output_world)) = cb.checkout_output() {
                     let out_extent_hint = output_world.extent_hint();
                     let progress_final = output_world.height() as _;
                     // iterate over image data.