/** * Copyright (C) 2026 Lynne * * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "libavutil/mem.h" #include "libavutil/refstruct.h" #include "../ops_internal.h" #include "../swscale_internal.h" #include "ops.h" static void ff_sws_vk_uninit(AVRefStructOpaque opaque, void *obj) { FFVulkanOpsCtx *s = obj; #if CONFIG_LIBSHADERC || CONFIG_LIBGLSLANG if (s->spvc) s->spvc->uninit(&s->spvc); #endif ff_vk_exec_pool_free(&s->vkctx, &s->e); ff_vk_uninit(&s->vkctx); } int ff_sws_vk_init(SwsContext *sws, AVBufferRef *dev_ref) { int err; SwsInternal *c = sws_internal(sws); if (!c->hw_priv) { c->hw_priv = av_refstruct_alloc_ext(sizeof(FFVulkanOpsCtx), 0, NULL, ff_sws_vk_uninit); if (!c->hw_priv) return AVERROR(ENOMEM); } FFVulkanOpsCtx *s = c->hw_priv; if (s->vkctx.device_ref && s->vkctx.device_ref->data != dev_ref->data) { /* Reinitialize with new context */ ff_vk_exec_pool_free(&s->vkctx, &s->e); ff_vk_uninit(&s->vkctx); } else if (s->vkctx.device_ref && s->vkctx.device_ref->data == dev_ref->data) { return 0; } err = ff_vk_init(&s->vkctx, sws, dev_ref, NULL); if (err < 0) return err; s->qf = ff_vk_qf_find(&s->vkctx, VK_QUEUE_COMPUTE_BIT, 0); if (!s->qf) { av_log(sws, AV_LOG_ERROR, "Device has no compute queues\n"); return AVERROR(ENOTSUP); } err = ff_vk_exec_pool_init(&s->vkctx, s->qf, &s->e, 1, 0, 0, 0, NULL); if (err < 0) return err; #if CONFIG_LIBSHADERC || CONFIG_LIBGLSLANG if (!s->spvc) { s->spvc = ff_vk_spirv_init(); if (!s->spvc) return AVERROR(ENOMEM); } #endif return 0; } typedef struct VulkanPriv { FFVulkanOpsCtx *s; FFVulkanShader shd; enum FFVkShaderRepFormat src_rep; enum FFVkShaderRepFormat dst_rep; } VulkanPriv; static void process(const SwsFrame *dst, const SwsFrame *src, int y, int h, const SwsPass *pass) { VulkanPriv *p = (VulkanPriv *) pass->priv; FFVkExecContext *ec = ff_vk_exec_get(&p->s->vkctx, &p->s->e); FFVulkanFunctions *vk = &p->s->vkctx.vkfn; ff_vk_exec_start(&p->s->vkctx, ec); AVFrame *src_f = (AVFrame *) src->avframe; AVFrame *dst_f = (AVFrame *) dst->avframe; ff_vk_exec_add_dep_frame(&p->s->vkctx, ec, src_f, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT); ff_vk_exec_add_dep_frame(&p->s->vkctx, ec, dst_f, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT); VkImageView src_views[AV_NUM_DATA_POINTERS]; VkImageView dst_views[AV_NUM_DATA_POINTERS]; ff_vk_create_imageviews(&p->s->vkctx, ec, src_views, src_f, p->src_rep); ff_vk_create_imageviews(&p->s->vkctx, ec, dst_views, dst_f, p->dst_rep); ff_vk_shader_update_img_array(&p->s->vkctx, ec, &p->shd, src_f, src_views, 0, 0, VK_IMAGE_LAYOUT_GENERAL, VK_NULL_HANDLE); ff_vk_shader_update_img_array(&p->s->vkctx, ec, &p->shd, dst_f, dst_views, 0, 1, VK_IMAGE_LAYOUT_GENERAL, VK_NULL_HANDLE); int nb_img_bar = 0; VkImageMemoryBarrier2 img_bar[8]; ff_vk_frame_barrier(&p->s->vkctx, ec, src_f, img_bar, &nb_img_bar, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_READ_BIT, VK_IMAGE_LAYOUT_GENERAL, VK_QUEUE_FAMILY_IGNORED); ff_vk_frame_barrier(&p->s->vkctx, ec, dst_f, img_bar, &nb_img_bar, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_IMAGE_LAYOUT_GENERAL, VK_QUEUE_FAMILY_IGNORED); vk->CmdPipelineBarrier2(ec->buf, &(VkDependencyInfo) { .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, .pImageMemoryBarriers = img_bar, .imageMemoryBarrierCount = nb_img_bar, }); ff_vk_exec_bind_shader(&p->s->vkctx, ec, &p->shd); vk->CmdDispatch(ec->buf, FFALIGN(src_f->width, p->shd.lg_size[0])/p->shd.lg_size[0], FFALIGN(src_f->height, p->shd.lg_size[1])/p->shd.lg_size[1], 1); ff_vk_exec_submit(&p->s->vkctx, ec); ff_vk_exec_wait(&p->s->vkctx, ec); } static void free_fn(void *priv) { VulkanPriv *p = priv; ff_vk_shader_free(&p->s->vkctx, &p->shd); av_free(priv); } #if CONFIG_LIBSHADERC || CONFIG_LIBGLSLANG static void add_desc_read_write(FFVulkanDescriptorSetBinding *out_desc, enum FFVkShaderRepFormat *out_rep, const SwsOp *op) { const char *img_type = op->type == SWS_PIXEL_F32 ? "rgba32f" : op->type == SWS_PIXEL_U32 ? "rgba32ui" : op->type == SWS_PIXEL_U16 ? "rgba16ui" : "rgba8ui"; *out_desc = (FFVulkanDescriptorSetBinding) { .name = op->op == SWS_OP_WRITE ? "dst_img" : "src_img", .type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, .mem_layout = img_type, .mem_quali = op->op == SWS_OP_WRITE ? "writeonly" : "readonly", .dimensions = 2, .elems = op->rw.packed ? 1 : op->rw.elems, .stages = VK_SHADER_STAGE_COMPUTE_BIT, }; *out_rep = op->type == SWS_PIXEL_F32 ? FF_VK_REP_FLOAT : FF_VK_REP_UINT; } static int add_ops_glsl(VulkanPriv *p, FFVulkanOpsCtx *s, SwsOpList *ops, FFVulkanShader *shd) { int err; uint8_t *spv_data; size_t spv_len; void *spv_opaque = NULL; /* Interlaced formats are not currently supported */ if (ops->src.interlaced || ops->dst.interlaced) return AVERROR(ENOTSUP); err = ff_vk_shader_init(&s->vkctx, shd, "sws_pass", VK_SHADER_STAGE_COMPUTE_BIT, NULL, 0, 32, 32, 1, 0); if (err < 0) return err; int nb_desc = 0; FFVulkanDescriptorSetBinding buf_desc[8]; const SwsOp *read = ff_sws_op_list_input(ops); const SwsOp *write = ff_sws_op_list_output(ops); if (read) add_desc_read_write(&buf_desc[nb_desc++], &p->src_rep, read); add_desc_read_write(&buf_desc[nb_desc++], &p->dst_rep, write); ff_vk_shader_add_descriptor_set(&s->vkctx, shd, buf_desc, nb_desc, 0, 0); GLSLC(0, void main() ); GLSLC(0, { ); GLSLC(1, ivec2 pos = ivec2(gl_GlobalInvocationID.xy); ); GLSLC(1, ivec2 size = imageSize(src_img[0]); ); GLSLC(1, if (any(greaterThanEqual(pos, size))) ); GLSLC(2, return; ); GLSLC(0, ); GLSLC(1, u8vec4 u8; ); GLSLC(1, u16vec4 u16; ); GLSLC(1, u32vec4 u32; ); GLSLC(1, f32vec4 f32; ); GLSLC(0, ); const char *type_name = ff_sws_pixel_type_name(ops->ops[0].type); for (int n = 0; n < ops->num_ops; n++) { const SwsOp *op = &ops->ops[n]; const char *type_v = op->type == SWS_PIXEL_F32 ? "f32vec4" : op->type == SWS_PIXEL_U32 ? "u32vec4" : op->type == SWS_PIXEL_U16 ? "u16vec4" : "u8vec4"; const char *type_s = op->type == SWS_PIXEL_F32 ? "float" : op->type == SWS_PIXEL_U32 ? "uint32_t" : op->type == SWS_PIXEL_U16 ? "uint16_t" : "uint8_t"; av_bprintf(&shd->src, " // %s\n", ff_sws_op_type_name(op->op)); switch (op->op) { case SWS_OP_READ: { if (op->rw.frac) { return AVERROR(ENOTSUP); } else if (op->rw.packed) { GLSLF(1, %s = %s(imageLoad(src_img[0], pos)); , type_name, type_v); } else { for (int i = 0; i < (op->rw.packed ? 1 : op->rw.elems); i++) GLSLF(1, %s.%c = %s(imageLoad(src_img[%i], pos)[0]); , type_name, "xyzw"[i], type_s, i); } break; } case SWS_OP_WRITE: { if (op->rw.frac) { return AVERROR(ENOTSUP); } else if (op->rw.packed) { GLSLF(1, imageStore(dst_img[0], pos, %s(%s)); , type_v, type_name); } else { for (int i = 0; i < (op->rw.packed ? 1 : op->rw.elems); i++) GLSLF(1, imageStore(dst_img[%i], pos, %s(%s[%i])); , i, type_v, type_name, i); } break; } case SWS_OP_SWIZZLE: { av_bprintf(&shd->src, " %s = %s.", type_name, type_name); for (int i = 0; i < 4; i++) av_bprintf(&shd->src, "%c", "xyzw"[op->swizzle.in[i]]); av_bprintf(&shd->src, ";\n"); break; } case SWS_OP_CLEAR: { for (int i = 0; i < 4; i++) { if (!op->c.q4[i].den) continue; av_bprintf(&shd->src, " %s.%c = %s(%i/%i%s);\n", type_name, "xyzw"[i], type_s, op->c.q4[i].num, op->c.q4[i].den, op->type == SWS_PIXEL_F32 ? ".0f" : ""); } break; } default: return AVERROR(ENOTSUP); } } GLSLC(0, } ); err = s->spvc->compile_shader(&s->vkctx, s->spvc, shd, &spv_data, &spv_len, "main", &spv_opaque); if (err < 0) return err; err = ff_vk_shader_link(&s->vkctx, shd, spv_data, spv_len, "main"); if (spv_opaque) s->spvc->free_shader(s->spvc, &spv_opaque); if (err < 0) return err; return 0; } #endif static int compile(SwsContext *sws, SwsOpList *ops, SwsCompiledOp *out) { int err; SwsInternal *c = sws_internal(sws); FFVulkanOpsCtx *s = c->hw_priv; if (!s) return AVERROR(ENOTSUP); VulkanPriv p = { .s = s, }; #if CONFIG_LIBSHADERC || CONFIG_LIBGLSLANG { err = add_ops_glsl(&p, s, ops, &p.shd); if (err < 0) return err; } #else return AVERROR(ENOTSUP); #endif err = ff_vk_shader_register_exec(&s->vkctx, &s->e, &p.shd); if (err < 0) return err; *out = (SwsCompiledOp) { .opaque = true, .func_opaque = process, .priv = av_memdup(&p, sizeof(p)), .free = free_fn, }; return 0; } const SwsOpBackend backend_vulkan = { .name = "vulkan", .compile = compile, .hw_format = AV_PIX_FMT_VULKAN, };