use std::str::FromStr;
use gtk::{gdk, gio, prelude::*};
use image::{ColorType, DynamicImage, ImageDecoder, ImageResult};
use matrix_sdk::{
attachment::{BaseImageInfo, BaseThumbnailInfo, Thumbnail},
media::{MediaFormat, MediaRequest, MediaThumbnailSettings, MediaThumbnailSize},
Client,
};
use ruma::{
api::client::media::get_content_thumbnail::v3::Method,
events::{
room::{
avatar::ImageInfo as AvatarImageInfo, ImageInfo, MediaSource as CommonMediaSource,
ThumbnailInfo,
},
sticker::StickerMediaSource,
},
OwnedMxcUri,
};
use tracing::warn;
use crate::{
components::AnimatedImagePaintable,
spawn_tokio,
utils::{matrix::MediaFileError, save_data_to_tmp_file},
DISABLE_GLYCIN_SANDBOX,
};
const THUMBNAIL_DEFAULT_WIDTH: u32 = 800;
const THUMBNAIL_DEFAULT_HEIGHT: u32 = 600;
const THUMBNAIL_DEFAULT_DIMENSIONS: ImageDimensions = ImageDimensions {
width: THUMBNAIL_DEFAULT_WIDTH,
height: THUMBNAIL_DEFAULT_HEIGHT,
};
const SVG_CONTENT_TYPE: &str = "image/svg+xml";
const WEBP_CONTENT_TYPE: &str = "image/webp";
const WEBP_DEFAULT_QUALITY: f32 = 60.0;
const THUMBNAIL_MAX_FILESIZE_THRESHOLD: u32 = 1024 * 1024;
const THUMBNAIL_DIMENSIONS_THRESHOLD: u32 = 200;
async fn image_reader(file: gio::File) -> Result<glycin::Image<'static>, glycin::ErrorCtx> {
let mut loader = glycin::Loader::new(file);
if DISABLE_GLYCIN_SANDBOX {
loader.sandbox_selector(glycin::SandboxSelector::NotSandboxed);
}
spawn_tokio!(async move { loader.load().await })
.await
.unwrap()
}
pub async fn load_image(
file: gio::File,
request_dimensions: Option<ImageDimensions>,
) -> Result<gdk::Paintable, glycin::ErrorCtx> {
let image = image_reader(file).await?;
let frame_request = request_dimensions.map(|request| {
let image_info = image.info();
let original_dimensions = ImageDimensions {
width: image_info.width,
height: image_info.height,
};
original_dimensions.to_image_loader_request(request)
});
let (image, first_frame) = spawn_tokio!(async move {
let first_frame = if let Some(frame_request) = frame_request {
image.specific_frame(frame_request).await?
} else {
image.next_frame().await?
};
Ok((image, first_frame))
})
.await
.unwrap()?;
let paintable = if first_frame.delay().is_some() {
AnimatedImagePaintable::new(image, first_frame).upcast()
} else {
first_frame.texture().upcast()
};
Ok(paintable)
}
pub enum ImageInfoLoader {
File(gio::File),
Texture(gdk::Texture),
}
impl ImageInfoLoader {
async fn into_first_frame(self) -> Option<Frame> {
match self {
Self::File(file) => {
let image_reader = image_reader(file).await.ok()?;
let handle = spawn_tokio!(async move { image_reader.next_frame().await });
Some(Frame::Glycin(handle.await.unwrap().ok()?))
}
Self::Texture(texture) => Some(Frame::Texture(gdk::TextureDownloader::new(&texture))),
}
}
pub async fn load_info(self) -> BaseImageInfo {
self.into_first_frame()
.await
.and_then(|f| f.dimensions())
.map(Into::into)
.unwrap_or_else(default_base_image_info)
}
pub async fn load_info_and_thumbnail(
self,
filesize: Option<u32>,
) -> (BaseImageInfo, Option<Thumbnail>) {
let Some(frame) = self.into_first_frame().await else {
return (default_base_image_info(), None);
};
let dimensions = frame.dimensions();
let info = dimensions
.map(Into::into)
.unwrap_or_else(default_base_image_info);
if !filesize_is_too_big(filesize)
&& !dimensions
.is_some_and(|d| d.should_resize_for_thumbnail(THUMBNAIL_DEFAULT_DIMENSIONS))
{
return (info, None);
}
let thumbnail = frame.generate_thumbnail();
(info, thumbnail)
}
}
impl From<gio::File> for ImageInfoLoader {
fn from(value: gio::File) -> Self {
Self::File(value)
}
}
impl From<gdk::Texture> for ImageInfoLoader {
fn from(value: gdk::Texture) -> Self {
Self::Texture(value)
}
}
enum Frame {
Glycin(glycin::Frame),
Texture(gdk::TextureDownloader),
}
impl Frame {
fn dimensions(&self) -> Option<ImageDimensions> {
let (width, height) = match self {
Self::Glycin(frame) => (frame.width(), frame.height()),
Self::Texture(downloader) => {
let texture = downloader.texture();
(
texture.width().try_into().ok()?,
texture.height().try_into().ok()?,
)
}
};
Some(ImageDimensions { width, height })
}
fn is_memory_format_supported(&self) -> bool {
match self {
Self::Glycin(frame) => {
matches!(
frame.memory_format(),
glycin::MemoryFormat::G8
| glycin::MemoryFormat::G8a8
| glycin::MemoryFormat::R8g8b8
| glycin::MemoryFormat::R8g8b8a8
| glycin::MemoryFormat::G16
| glycin::MemoryFormat::G16a16
| glycin::MemoryFormat::R16g16b16
| glycin::MemoryFormat::R16g16b16a16
| glycin::MemoryFormat::R32g32b32Float
| glycin::MemoryFormat::R32g32b32a32Float
)
}
Self::Texture(downloader) => {
matches!(
downloader.format(),
gdk::MemoryFormat::G8
| gdk::MemoryFormat::G8a8
| gdk::MemoryFormat::R8g8b8
| gdk::MemoryFormat::R8g8b8a8
| gdk::MemoryFormat::G16
| gdk::MemoryFormat::G16a16
| gdk::MemoryFormat::R16g16b16
| gdk::MemoryFormat::R16g16b16a16
| gdk::MemoryFormat::R32g32b32Float
| gdk::MemoryFormat::R32g32b32a32Float
)
}
}
}
fn generate_thumbnail(self) -> Option<Thumbnail> {
if !self.is_memory_format_supported() {
return None;
}
let image = DynamicImage::from_decoder(self).ok()?;
let thumbnail = image.thumbnail(THUMBNAIL_DEFAULT_WIDTH, THUMBNAIL_DEFAULT_HEIGHT);
prepare_thumbnail_for_sending(thumbnail)
}
}
impl ImageDecoder for Frame {
fn dimensions(&self) -> (u32, u32) {
let (width, height) = self.dimensions().map(|d| (d.width, d.height)).unzip();
(width.unwrap_or(0), height.unwrap_or(0))
}
fn color_type(&self) -> ColorType {
match self {
Self::Glycin(frame) => match frame.memory_format() {
glycin::MemoryFormat::G8 => ColorType::L8,
glycin::MemoryFormat::G8a8 => ColorType::La8,
glycin::MemoryFormat::R8g8b8 => ColorType::Rgb8,
glycin::MemoryFormat::R8g8b8a8 => ColorType::Rgba8,
glycin::MemoryFormat::G16 => ColorType::L16,
glycin::MemoryFormat::G16a16 => ColorType::La16,
glycin::MemoryFormat::R16g16b16 => ColorType::Rgb16,
glycin::MemoryFormat::R16g16b16a16 => ColorType::Rgba16,
glycin::MemoryFormat::R32g32b32Float => ColorType::Rgb32F,
glycin::MemoryFormat::R32g32b32a32Float => ColorType::Rgba32F,
_ => unimplemented!(),
},
Self::Texture(downloader) => match downloader.format() {
gdk::MemoryFormat::G8 => ColorType::L8,
gdk::MemoryFormat::G8a8 => ColorType::La8,
gdk::MemoryFormat::R8g8b8 => ColorType::Rgb8,
gdk::MemoryFormat::R8g8b8a8 => ColorType::Rgba8,
gdk::MemoryFormat::G16 => ColorType::L16,
gdk::MemoryFormat::G16a16 => ColorType::La16,
gdk::MemoryFormat::R16g16b16 => ColorType::Rgb16,
gdk::MemoryFormat::R16g16b16a16 => ColorType::Rgba16,
gdk::MemoryFormat::R32g32b32Float => ColorType::Rgb32F,
gdk::MemoryFormat::R32g32b32a32Float => ColorType::Rgba32F,
_ => unimplemented!(),
},
}
}
fn read_image(self, buf: &mut [u8]) -> ImageResult<()>
where
Self: Sized,
{
let bytes = match &self {
Self::Glycin(frame) => frame.buf_bytes(),
Self::Texture(texture) => texture.download_bytes().0,
};
buf.copy_from_slice(&bytes);
Ok(())
}
fn read_image_boxed(self: Box<Self>, _buf: &mut [u8]) -> ImageResult<()> {
unimplemented!()
}
}
#[derive(Debug, Clone, Copy)]
pub struct ImageDimensions {
pub width: u32,
pub height: u32,
}
impl ImageDimensions {
fn from_options(width: Option<u32>, height: Option<u32>) -> Option<Self> {
Some(Self {
width: width?,
height: height?,
})
}
pub fn is_bigger_than(&self, other: ImageDimensions) -> bool {
self.width >= other.width || self.height >= other.height
}
pub fn should_resize_for_thumbnail(&self, thumbnail_dimensions: ImageDimensions) -> bool {
self.is_bigger_than(thumbnail_dimensions.increase_by(THUMBNAIL_DIMENSIONS_THRESHOLD))
}
pub const fn increase_by(mut self, value: u32) -> Self {
self.width = self.width.saturating_add(value);
self.height = self.height.saturating_add(value);
self
}
pub fn resize(self, requested_dimensions: ImageDimensions, strategy: ResizeStrategy) -> Self {
let w_ratio = self.width as f64 / requested_dimensions.width as f64;
let h_ratio = self.height as f64 / requested_dimensions.height as f64;
let resize_from_width = match strategy {
ResizeStrategy::Contain => w_ratio > h_ratio,
ResizeStrategy::Cover => w_ratio < h_ratio,
};
let (width, height) = if resize_from_width {
let new_height = self.height as f64 / w_ratio;
(requested_dimensions.width, new_height as u32)
} else {
let new_width = self.width as f64 / h_ratio;
(new_width as u32, requested_dimensions.height)
};
Self { width, height }
}
pub fn resize_for_thumbnail(self) -> Option<Self> {
let thumbnail_dimensions = THUMBNAIL_DEFAULT_DIMENSIONS;
if !self.should_resize_for_thumbnail(thumbnail_dimensions) {
return None;
}
Some(self.resize(thumbnail_dimensions, ResizeStrategy::Contain))
}
pub fn to_image_loader_request(
self,
requested_dimensions: ImageDimensions,
) -> glycin::FrameRequest {
let resized_dimensions = self.resize(requested_dimensions, ResizeStrategy::Cover);
glycin::FrameRequest::new().scale(resized_dimensions.width, resized_dimensions.height)
}
}
impl From<ImageDimensions> for BaseImageInfo {
fn from(value: ImageDimensions) -> Self {
let ImageDimensions { width, height } = value;
BaseImageInfo {
height: Some(height.into()),
width: Some(width.into()),
size: None,
blurhash: None,
}
}
}
fn default_base_image_info() -> BaseImageInfo {
BaseImageInfo {
height: None,
width: None,
size: None,
blurhash: None,
}
}
#[derive(Debug, Clone, Copy)]
pub enum ResizeStrategy {
Contain,
Cover,
}
pub(super) fn prepare_thumbnail_for_sending(thumbnail: image::DynamicImage) -> Option<Thumbnail> {
let thumbnail: DynamicImage = match &thumbnail {
DynamicImage::ImageLuma8(_)
| DynamicImage::ImageRgb8(_)
| DynamicImage::ImageLuma16(_)
| DynamicImage::ImageRgb16(_)
| DynamicImage::ImageRgb32F(_) => thumbnail.into_rgb8().into(),
DynamicImage::ImageLumaA8(_)
| DynamicImage::ImageRgba8(_)
| DynamicImage::ImageLumaA16(_)
| DynamicImage::ImageRgba16(_)
| DynamicImage::ImageRgba32F(_) => thumbnail.into_rgba8().into(),
_ => return None,
};
let encoder = webp::Encoder::from_image(&thumbnail).ok()?;
let thumbnail_bytes = encoder.encode(WEBP_DEFAULT_QUALITY).to_vec();
let thumbnail_content_type =
mime::Mime::from_str(WEBP_CONTENT_TYPE).expect("content type should be valid");
let thumbnail_info = BaseThumbnailInfo {
width: Some(thumbnail.width().into()),
height: Some(thumbnail.height().into()),
size: thumbnail_bytes.len().try_into().ok(),
};
Some(Thumbnail {
data: thumbnail_bytes,
content_type: thumbnail_content_type,
info: Some(thumbnail_info),
})
}
#[derive(Debug, Clone, Copy)]
pub struct ThumbnailDownloader<'a> {
pub main: ImageSource<'a>,
pub alt: Option<ImageSource<'a>>,
}
impl<'a> ThumbnailDownloader<'a> {
pub async fn download_to_file(
self,
client: &Client,
settings: ThumbnailSettings,
) -> Result<gio::File, MediaFileError> {
let source = if let Some(alt) = self.alt {
if !self.main.can_be_thumbnailed()
&& (filesize_is_too_big(self.main.filesize())
|| alt
.dimensions()
.is_some_and(|d| d.is_bigger_than(settings.dimensions)))
{
alt
} else {
self.main
}
} else {
self.main
};
let data = if source.should_thumbnail(settings.prefer_thumbnail, settings.dimensions) {
let media = client.media();
let request = MediaRequest {
source: source.source.to_common_media_source(),
format: MediaFormat::Thumbnail(settings.into()),
};
let handle = spawn_tokio!(async move { media.get_media_content(&request, true).await });
match handle.await.unwrap() {
Ok(data) => Some(data),
Err(error) => {
warn!("Could not retrieve media thumbnail: {error}");
None
}
}
} else {
None
};
let data = if let Some(data) = data {
data
} else {
let media = client.media();
let request = MediaRequest {
source: source.source.to_common_media_source(),
format: MediaFormat::File,
};
spawn_tokio!(async move { media.get_media_content(&request, true).await })
.await
.unwrap()?
};
Ok(save_data_to_tmp_file(&data)?)
}
}
#[derive(Debug, Clone, Copy)]
pub struct ImageSource<'a> {
pub source: MediaSource<'a>,
pub info: Option<ImageSourceInfo<'a>>,
}
impl<'a> ImageSource<'a> {
fn should_thumbnail(
&self,
prefer_thumbnail: bool,
thumbnail_dimensions: ImageDimensions,
) -> bool {
if !self.can_be_thumbnailed() {
return false;
}
let dimensions = self.dimensions();
if prefer_thumbnail && dimensions.is_none() {
return true;
}
dimensions.is_some_and(|d| d.should_resize_for_thumbnail(thumbnail_dimensions))
|| filesize_is_too_big(self.filesize())
}
fn can_be_thumbnailed(&self) -> bool {
!self.source.is_encrypted()
&& !self
.info
.is_some_and(|i| i.mimetype.is_some_and(|m| m == SVG_CONTENT_TYPE))
}
fn filesize(&self) -> Option<u32> {
self.info.and_then(|i| i.size)
}
fn dimensions(&self) -> Option<ImageDimensions> {
self.info.and_then(|i| i.dimensions)
}
}
fn filesize_is_too_big(filesize: Option<u32>) -> bool {
filesize.is_some_and(|s| s > THUMBNAIL_MAX_FILESIZE_THRESHOLD)
}
#[derive(Debug, Clone, Copy)]
pub enum MediaSource<'a> {
Common(&'a CommonMediaSource),
Sticker(&'a StickerMediaSource),
Uri(&'a OwnedMxcUri),
}
impl<'a> MediaSource<'a> {
fn is_encrypted(&self) -> bool {
match self {
Self::Common(source) => matches!(source, CommonMediaSource::Encrypted(_)),
Self::Sticker(source) => matches!(source, StickerMediaSource::Encrypted(_)),
Self::Uri(_) => false,
}
}
fn to_common_media_source(self) -> CommonMediaSource {
match self {
Self::Common(source) => source.clone(),
Self::Sticker(source) => source.clone().into(),
Self::Uri(uri) => CommonMediaSource::Plain(uri.clone()),
}
}
}
impl<'a> From<&'a CommonMediaSource> for MediaSource<'a> {
fn from(value: &'a CommonMediaSource) -> Self {
Self::Common(value)
}
}
impl<'a> From<&'a StickerMediaSource> for MediaSource<'a> {
fn from(value: &'a StickerMediaSource) -> Self {
Self::Sticker(value)
}
}
impl<'a> From<&'a OwnedMxcUri> for MediaSource<'a> {
fn from(value: &'a OwnedMxcUri) -> Self {
Self::Uri(value)
}
}
#[derive(Debug, Clone, Copy, Default)]
pub struct ImageSourceInfo<'a> {
dimensions: Option<ImageDimensions>,
mimetype: Option<&'a str>,
size: Option<u32>,
}
impl<'a> From<&'a ImageInfo> for ImageSourceInfo<'a> {
fn from(value: &'a ImageInfo) -> Self {
Self {
dimensions: ImageDimensions::from_options(
value.width.and_then(|u| u.try_into().ok()),
value.height.and_then(|u| u.try_into().ok()),
),
mimetype: value.mimetype.as_deref(),
size: value.size.and_then(|u| u.try_into().ok()),
}
}
}
impl<'a> From<&'a ThumbnailInfo> for ImageSourceInfo<'a> {
fn from(value: &'a ThumbnailInfo) -> Self {
Self {
dimensions: ImageDimensions::from_options(
value.width.and_then(|u| u.try_into().ok()),
value.height.and_then(|u| u.try_into().ok()),
),
mimetype: value.mimetype.as_deref(),
size: value.size.and_then(|u| u.try_into().ok()),
}
}
}
impl<'a> From<&'a AvatarImageInfo> for ImageSourceInfo<'a> {
fn from(value: &'a AvatarImageInfo) -> Self {
Self {
dimensions: ImageDimensions::from_options(
value.width.and_then(|u| u.try_into().ok()),
value.height.and_then(|u| u.try_into().ok()),
),
mimetype: value.mimetype.as_deref(),
size: value.size.and_then(|u| u.try_into().ok()),
}
}
}
#[derive(Debug, Clone)]
pub struct ThumbnailSettings {
pub dimensions: ImageDimensions,
pub method: Method,
pub animated: bool,
pub prefer_thumbnail: bool,
}
impl From<ThumbnailSettings> for MediaThumbnailSettings {
fn from(value: ThumbnailSettings) -> Self {
let ThumbnailSettings {
dimensions,
method,
animated,
..
} = value;
MediaThumbnailSettings {
size: MediaThumbnailSize {
method,
width: dimensions.width.into(),
height: dimensions.height.into(),
},
animated,
}
}
}