Skip to content

ggml : support YaRN truncation config on openvino, sycl, vulkan, webgpu, rpc #30802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
tokinasin wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

ggml : support YaRN truncation config on openvino, sycl, vulkan, webgpu, rpc #30802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4bc2928
openvino : support YaRN truncation config
tokinasin Oct 19, 2026
cdce86a
sycl : support YaRN truncation config
tokinasin Oct 19, 2026
ad39eb1
vulkan : support YaRN truncation config
tokinasin Oct 19, 2026
17cca73
webgpu : support YaRN truncation config
tokinasin Oct 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 5 additions & 20 deletions ggml/src/ggml-openvino/openvino/utils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
#include "utils.h"

#include "ggml-impl.h"

#include <cmath>
#include <cstddef>
#include <ctime>
Expand Down Expand Up @@ -90,24 +92,6 @@ ov::Output<ov::Node> rope_yarn_ramp_mix(int num_elements, const float corr_dims[
return ramp_mix;
}

float ggml_rope_yarn_corr_dim(int n_dims, int n_ctx_orig, float n_rot, float base) {
#ifndef M_PI
# define M_PI 3.14159265358979323846
#endif
return n_dims * logf(n_ctx_orig / (n_rot * 2 * (float) M_PI)) / (2 * logf(base));
}

void ggml_rope_yarn_corr_dims(int n_dims,
int n_ctx_orig,
float freq_base,
float beta_fast,
float beta_slow,
float dims[2]) {
float start = floorf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_fast, freq_base));
float end = ceilf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_slow, freq_base));
dims[0] = std::max(0.0f, start);
dims[1] = std::min(static_cast<float>(n_dims - 1), end);
}
} // namespace

std::pair<ov::Output<Node>, ov::Output<Node>> make_sin_cos(int32_t * rope_params,
Expand Down Expand Up @@ -154,6 +138,7 @@ std::pair<ov::Output<Node>, ov::Output<Node>> make_sin_cos(int32_t * rope_params
float beta_slow;
const int n_dims = rope_params[1];
const size_t n_dims_half = n_dims >> 1;
const bool truncate = rope_params[3] == 0;
const int n_ctx_orig = rope_params[4];
memcpy(&freq_base, rope_params + 5, sizeof(float));
memcpy(&freq_scale, rope_params + 6, sizeof(float));
Expand Down Expand Up @@ -248,7 +233,7 @@ std::pair<ov::Output<Node>, ov::Output<Node>> make_sin_cos(int32_t * rope_params
theta = theta_interp;
} else {
float corr_dims[2];
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, truncate, corr_dims);
auto ramp_mix = rope_yarn_ramp_mix(num_pairs, corr_dims, ext_factor);
Output<Node> one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {1.0f});
auto one_minus_ramp = std::make_shared<ov::op::v1::Subtract>(one, ramp_mix);
Expand All @@ -261,7 +246,7 @@ std::pair<ov::Output<Node>, ov::Output<Node>> make_sin_cos(int32_t * rope_params
std::vector<float> factor(n_dims_half);
Output<Node> freq_factors;
float corr_dims[2];
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, truncate, corr_dims);
factor[0] = 1.0f;
for (size_t i = 1; i < factor.size(); i++) {
factor[i] = theta_scale * factor[i - 1];
Expand Down
2 changes: 1 addition & 1 deletion ggml/src/ggml-sycl/rope.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -547,7 +547,7 @@ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst,

rope_corr_dims corr_dims;
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast,
beta_slow, corr_dims.v);
beta_slow, ggml_rope_get_truncate(dst), corr_dims.v);

// compute
if (is_neox) {
Expand Down
2 changes: 1 addition & 1 deletion ggml/src/ggml-vulkan/ggml-vulkan-debug.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1054,7 +1054,6 @@ void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph
} else if (tensor->op == GGML_OP_ROPE || tensor->op == GGML_OP_ROPE_BACK) {
const int n_dims = ((int32_t *) tensor->op_params)[1];
const int mode = ((int32_t *) tensor->op_params)[2];
//const int n_ctx_ggml = ((int32_t *) tensor->op_params)[3];
const int n_ctx_orig_ggml = ((int32_t *) tensor->op_params)[4];
const float freq_base = ((float *) tensor->op_params)[5];
const float freq_scale = ((float *) tensor->op_params)[6];
Expand All @@ -1076,6 +1075,7 @@ void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph
tensor_clone = ggml_rope_ext_back(ggml_ctx, src_clone[0], src_clone[1], src_clone[2], n_dims, mode, n_ctx_orig_ggml, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
}
}
ggml_rope_set_truncate(tensor_clone, ggml_rope_get_truncate(tensor));
const int n_offs = ((int32_t *) tensor->op_params)[15];
if (n_offs != 0) {
tensor_clone = ggml_rope_set_offset(tensor_clone, n_offs);
Expand Down
17 changes: 1 addition & 16 deletions ggml/src/ggml-vulkan/ggml-vulkan.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10786,7 +10786,6 @@ static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor *
const int n_dims = ((const int32_t *) dst->op_params)[1];
const int mode = ((const int32_t *) dst->op_params)[2];
const int n_offs = ((const int32_t *) dst->op_params)[15];
// const int n_ctx = ((const int32_t *) dst->op_params)[3];
const int n_ctx_orig = ((const int32_t *) dst->op_params)[4];
const float freq_base = ((const float *) dst->op_params)[5];
const float freq_scale = ((const float *) dst->op_params)[6];
Expand All @@ -10802,7 +10801,7 @@ static vk_op_rope_push_constants ggml_vk_make_rope_constants(const ggml_tensor *
const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE;

float corr_dims[2];
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, ggml_rope_get_truncate(dst), corr_dims);

const float theta_scale = powf(freq_base, -2.0f/n_dims);

Expand Down Expand Up @@ -11286,20 +11285,6 @@ void ggml_vk_rope(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_
const ggml_tensor * src1 = dst->src[1];
const ggml_tensor * src2 = dst->src[2];
const ggml_tensor * src3 = nullptr;
const int n_dims = ((int32_t *) dst->op_params)[1];
const int mode = ((int32_t *) dst->op_params)[2];
// const int n_ctx = ((int32_t *) dst->op_params)[3];
const int n_ctx_orig = ((int32_t *) dst->op_params)[4];
const float freq_base = ((float *) dst->op_params)[5];
const float beta_fast = ((float *) dst->op_params)[9];
const float beta_slow = ((float *) dst->op_params)[10];
int sections[4] {};
if (mode & GGML_ROPE_TYPE_MROPE) {
memcpy(sections, (int32_t *) dst->op_params + 11, sizeof(int)*4);
}

float corr_dims[2];
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);

uint32_t set_rows_stride = 0;
// Fused rope + view + set_rows passes the set_rows destination stride in set_rows_stride
Expand Down
3 changes: 2 additions & 1 deletion ggml/src/ggml-webgpu/ggml-webgpu.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2780,7 +2780,8 @@ static webgpu_encoded_op ggml_webgpu_rope(webgpu_context & ctx,
float theta_scale = powf(freq_base, -2.0f / n_dims);

float corr_dims[2];
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, ggml_rope_get_truncate(dst),
corr_dims);

std::vector<uint32_t> params = {
(uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src0) / ggml_type_size(src0->type)),
Expand Down