1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
//! Collection of methods for videos.

use std::sync::{Arc, Mutex};

use futures_channel::oneshot;
use gst::prelude::*;
use gst_video::prelude::*;
use gtk::{gio, glib, glib::clone, prelude::*};
use image::GenericImageView;
use matrix_sdk::attachment::{BaseVideoInfo, Thumbnail};
use tracing::warn;

use super::{
    image::{prepare_thumbnail_for_sending, ImageDimensions},
    load_gstreamer_media_info,
};

/// A channel sender to send the result of a video thumbnail.
type ThumbnailResultSender = oneshot::Sender<Result<Thumbnail, ()>>;

/// Load information and try to generate a thumbnail for the video in the given
/// file.
pub async fn load_video_info(file: &gio::File) -> (BaseVideoInfo, Option<Thumbnail>) {
    let mut info = BaseVideoInfo {
        duration: None,
        width: None,
        height: None,
        size: None,
        blurhash: None,
    };

    let Some(media_info) = load_gstreamer_media_info(file).await else {
        return (info, None);
    };

    info.duration = media_info.duration().map(Into::into);

    if let Some(stream_info) = media_info
        .video_streams()
        .first()
        .and_then(|s| s.downcast_ref::<gst_pbutils::DiscovererVideoInfo>())
    {
        info.width = Some(stream_info.width().into());
        info.height = Some(stream_info.height().into());
    }

    let thumbnail = generate_video_thumbnail(file).await;

    (info, thumbnail)
}

/// Generate a thumbnail for the video in the given file.
async fn generate_video_thumbnail(file: &gio::File) -> Option<Thumbnail> {
    let (sender, receiver) = oneshot::channel();
    let sender = Arc::new(Mutex::new(Some(sender)));

    let pipeline = match create_thumbnailer_pipeline(&file.uri(), sender.clone()) {
        Ok(pipeline) => pipeline,
        Err(error) => {
            warn!("Could not create pipeline for video thumbnail: {error}");
            return None;
        }
    };

    if pipeline.set_state(gst::State::Paused).is_err() {
        warn!("Could not initialize pipeline for video thumbnail");
        return None;
    }

    let bus = pipeline.bus().expect("Pipeline has a bus");

    let mut started = false;
    let _bus_guard = bus
        .add_watch(clone!(
            #[weak]
            pipeline,
            #[upgrade_or]
            glib::ControlFlow::Break,
            move |_, message| {
                match message.view() {
                    gst::MessageView::AsyncDone(_) => {
                        if !started {
                            // AsyncDone means that the pipeline has started now.
                            if pipeline.set_state(gst::State::Playing).is_err() {
                                warn!("Could not start pipeline for video thumbnail");
                                send_video_thumbnail_result(&sender, Err(()));

                                return glib::ControlFlow::Break;
                            };

                            started = true;
                        }

                        glib::ControlFlow::Continue
                    }
                    gst::MessageView::Eos(_) => {
                        // We have the thumbnail or we cannot have one.
                        glib::ControlFlow::Break
                    }
                    gst::MessageView::Error(error) => {
                        warn!("Could not generate video thumbnail: {error}");
                        send_video_thumbnail_result(&sender, Err(()));

                        glib::ControlFlow::Break
                    }
                    _ => glib::ControlFlow::Continue,
                }
            }
        ))
        .expect("Setting bus watch succeeds");

    let thumbnail = receiver.await;

    // Clean up.
    let _ = pipeline.set_state(gst::State::Null);
    bus.set_flushing(true);

    thumbnail.ok().transpose().ok().flatten()
}

/// Create a GStreamer pipeline to get a thumbnail of the first frame.
fn create_thumbnailer_pipeline(
    uri: &str,
    sender: Arc<Mutex<Option<ThumbnailResultSender>>>,
) -> Result<gst::Pipeline, glib::Error> {
    // Create our pipeline from a pipeline description string.
    let pipeline = gst::parse::launch(&format!(
        "uridecodebin uri={uri} ! videoconvert ! appsink name=sink"
    ))?
    .downcast::<gst::Pipeline>()
    .expect("Element is a pipeline");

    let appsink = pipeline
        .by_name("sink")
        .expect("Sink element is in the pipeline")
        .downcast::<gst_app::AppSink>()
        .expect("Sink element is an appsink");

    // Don't synchronize on the clock, we only want a snapshot asap.
    appsink.set_property("sync", false);

    // Tell the appsink what format we want, for simplicity we only accept 8-bit
    // RGB.
    appsink.set_caps(Some(
        &gst_video::VideoCapsBuilder::new()
            .format(gst_video::VideoFormat::Rgbx)
            .build(),
    ));

    let mut got_snapshot = false;

    // Listen to callbacks to get the data.
    appsink.set_callbacks(
        gst_app::AppSinkCallbacks::builder()
            .new_sample(move |appsink| {
                // Pull the sample out of the buffer.
                let sample = appsink.pull_sample().map_err(|_| gst::FlowError::Eos)?;
                let Some(buffer) = sample.buffer() else {
                    warn!("Could not get buffer from appsink");
                    send_video_thumbnail_result(&sender, Err(()));

                    return Err(gst::FlowError::Error);
                };

                // Make sure that we only get a single buffer.
                if got_snapshot {
                    return Err(gst::FlowError::Eos);
                }
                got_snapshot = true;

                let Some(caps) = sample.caps() else {
                    warn!("Got video sample without caps");
                    send_video_thumbnail_result(&sender, Err(()));

                    return Err(gst::FlowError::Error);
                };
                let Ok(info) = gst_video::VideoInfo::from_caps(caps) else {
                    warn!("Could not parse video caps");
                    send_video_thumbnail_result(&sender, Err(()));

                    return Err(gst::FlowError::Error);
                };

                let frame = gst_video::VideoFrameRef::from_buffer_ref_readable(buffer, &info)
                    .map_err(|_| {
                        warn!("Could not map video buffer readable");
                        send_video_thumbnail_result(&sender, Err(()));

                        gst::FlowError::Error
                    })?;

                // Create a FlatSamples around the borrowed video frame data from GStreamer with
                // the correct stride.
                let img = image::FlatSamples::<&[u8]> {
                    samples: frame.plane_data(0).unwrap(),
                    layout: image::flat::SampleLayout {
                        channels: 3,       // RGB
                        channel_stride: 1, // 1 byte from component to component
                        width: frame.width(),
                        width_stride: 4, // 4 bytes from pixel to pixel
                        height: frame.height(),
                        height_stride: frame.plane_stride()[0] as usize, // stride from line to line
                    },
                    color_hint: Some(image::ColorType::Rgb8),
                };

                let Ok(view) = img.as_view::<image::Rgb<u8>>() else {
                    warn!("Could not parse frame as view");
                    send_video_thumbnail_result(&sender, Err(()));

                    return Err(gst::FlowError::Error);
                };

                // Reduce the dimensions if the thumbnail is bigger than the wanted size.
                let dimensions = ImageDimensions {
                    width: frame.width() * info.par().numer() as u32,
                    height: frame.height() * info.par().denom() as u32,
                };

                let thumbnail = if let Some(target_dimensions) = dimensions.resize_for_thumbnail() {
                    image::imageops::thumbnail(
                        &view,
                        target_dimensions.width,
                        target_dimensions.height,
                    )
                } else {
                    image::ImageBuffer::from_fn(view.width(), view.height(), |x, y| {
                        view.get_pixel(x, y)
                    })
                };

                // Prepare it.
                if let Some(thumbnail) = prepare_thumbnail_for_sending(thumbnail.into()) {
                    send_video_thumbnail_result(&sender, Ok(thumbnail));

                    Err(gst::FlowError::Eos)
                } else {
                    warn!("Failed to convert video thumbnail");
                    send_video_thumbnail_result(&sender, Err(()));

                    Err(gst::FlowError::Error)
                }
            })
            .build(),
    );

    Ok(pipeline)
}

/// Try to send the given video thumbnail result through the given sender.
fn send_video_thumbnail_result(
    sender: &Mutex<Option<ThumbnailResultSender>>,
    result: Result<Thumbnail, ()>,
) {
    let mut sender = match sender.lock() {
        Ok(sender) => sender,
        Err(error) => {
            warn!("Failed to lock video thumbnail mutex: {error}");
            return;
        }
    };

    if let Some(sender) = sender.take() {
        if sender.send(result).is_err() {
            warn!("Failed to send video thumbnail result through channel");
        }
    }
}