Plan a score again when the planned one cannot be sung (plan_retries)

docs/ARCHITECTURE.md

@@ -591,6 +591,7 @@ the server rejects a request with neither.
     "source_tokens":   "",
     "source_start":    0.0,
     "source_end":      -1.0,
+    "plan_retries":    2,
     "abc_sampling":      { "temperature": 0.7, "top_p": 0.9,  "top_k": 30,  "repetition_penalty": 1.005, "penalty_window": 100, "min_tokens": 32,  "max_tokens": 4096 },
     "semantic_sampling": { "temperature": 1.0, "top_p": 0.95, "top_k": 100, "repetition_penalty": 1.2,   "penalty_window": 50,  "min_tokens": 200, "max_tokens": 9000 }
 }
@@ -692,6 +693,21 @@ repetition penalty positive, penalty window in [1, 100], `min_tokens`
 between 0 and `max_tokens`, `max_tokens` at least 1. A preset outside the
 bounds is a 400 from the server and a FATAL from the CLI.
 
+A planned score nothing can be sung from stops the request before the
+semantic stage (`src/score-check.h`): a header without a well-formed `M:`,
+`L:` or `K:`, no voice with bars, or colons inside the note runs outside
+chord symbols, inline fields and repeat bars. The log shows the start of the
+score in printable ASCII and a FATAL naming what is wrong; `yue-plan` only
+warns.
+
+**`plan_retries`** (int, default `2`, bound `8`)
+How many times a planned score that cannot be sung is planned again before
+the request stops. Song `i`'s attempt `k` draws with `lm_seed + i + k *
+lm_batch_size`, a seed no other song of the batch draws, so the same request
+replays to the same score; the replay request of the track carries the score
+that was sung. Outside the bound is a 400 from the server and a FATAL from
+the CLI.
+
 ## yue-plan reference
 
 Runs the first autoregressive stage alone and writes the composition the
@@ -806,9 +822,9 @@ POST /synth                     Submit a generation job, returns job ID
   response: {"id":"1a2b..."}
   400 on malformed JSON, unknown cot mode, unknown output_format,
   steps < 1, lm_batch_size outside [1, --max-batch], synth_batch_size
-  outside [1, 9], a sampling preset outside the protocol bounds, an
-  adapter missing from the --adapters directory, or both semantic_tokens
-  and source_tokens
+  outside [1, 9], plan_retries outside [0, 8], a sampling preset outside
+  the protocol bounds, an adapter missing from the --adapters directory, or
+  both semantic_tokens and source_tokens
 
 POST /transcribe                Submit a transcription job, returns job ID
   body: multipart/form-data, an "audio" part (WAV or MP3) and an optional

src/pipeline.h

@@ -22,6 +22,7 @@
 #include "model-store.h"
 #include "nar.h"
 #include "request.h"
+#include "score-check.h"
 #include "timer.h"
 #include "torch-cpu-rng.h"
 #include "vae-enc.h"
@@ -434,6 +435,10 @@ static bool pipeline_generate(Yue2Pipeline *          p,
     if (!yue2_sampling_valid(r.abc_sampling, "abc") || !yue2_sampling_valid(r.semantic_sampling, "semantic")) {
         return false;
     }
+    if (r.plan_retries < 0 || r.plan_retries > 8) {
+        fprintf(stderr, "[Pipeline] FATAL: plan_retries must be between 0 and 8\n");
+        return false;
+    }
     std::vector<AdapterUse> ar_adapters, nar_adapters;
     if (!pipeline_adapters(p, r, &ar_adapters, &nar_adapters)) {
         return false;
@@ -498,6 +503,36 @@ static bool pipeline_generate(Yue2Pipeline *          p,
             abc_ids[i]   = plans[i].tokens;
             scores[i]    = bpe_decode(tok, abc_ids[i]);
             truncated[i] = plans[i].truncated;
+            // a plan nothing can be sung from is planned again on seeds no
+            // other song of the batch draws, and stops the song before the
+            // long stage once the retries are spent
+            std::string problem = yue2_score_problem(scores[i]);
+            for (int attempt = 1; !problem.empty() && attempt <= r.plan_retries; attempt++) {
+                const int64_t seed = r.lm_seed + i + (int64_t) attempt * B;
+                fprintf(stderr,
+                        "[Pipeline] score of song %d cannot be sung: %s; planning it again with lm_seed %lld "
+                        "(%d of %d)\n",
+                        i + 1, problem.c_str(), (long long) seed, attempt, r.plan_retries);
+                std::vector<Yue2Generation> again;
+                if (!yue2_generate(lm, &p->kv, { open }, {}, 1.0f, r.abc_sampling, seed, YUE2_PHASE_ABC, &again,
+                                   cancelled, cancel_data)) {
+                    return false;
+                }
+                abc_ids[i]   = again[0].tokens;
+                scores[i]    = bpe_decode(tok, abc_ids[i]);
+                truncated[i] = again[0].truncated;
+                problem      = yue2_score_problem(scores[i]);
+            }
+            if (!problem.empty()) {
+                fprintf(stderr, "[Pipeline] score of song %d as written:\n%s\n", i + 1,
+                        yue2_score_excerpt(scores[i], 800).c_str());
+                fprintf(stderr, "[Pipeline] FATAL: the score the model wrote for song %d cannot be sung: %s. %s\n",
+                        i + 1, problem.c_str(),
+                        r.adapters.empty() ? "Draw another seed."
+                                           : "Adapters past their limits on the score half do this: lower their "
+                                             "strengths or draw another seed.");
+                return false;
+            }
         }
     }
 

src/request.cpp

@@ -32,6 +32,7 @@ void request_init(Yue2Request * r) {
     r->semantic_tokens  = "";
 
     r->abc_sampling      = YUE2_ABC_SAMPLING;
+    r->plan_retries      = 2;
     r->semantic_sampling = YUE2_SEMANTIC_SAMPLING;
 
     r->output_format = OUTPUT_FORMAT_MP3;
@@ -153,6 +154,9 @@ static void request_parse_obj(yyjson_val * obj, Yue2Request * r) {
         r->synth_batch_size = yyjson_get_int(v);
     }
     parse_sampling(obj, "abc_sampling", &r->abc_sampling);
+    if ((v = yyjson_obj_get(obj, "plan_retries")) && yyjson_is_int(v)) {
+        r->plan_retries = yyjson_get_int(v);
+    }
     parse_sampling(obj, "semantic_sampling", &r->semantic_sampling);
     if ((v = yyjson_obj_get(obj, "semantic_tokens")) && yyjson_is_str(v)) {
         r->semantic_tokens = yy_str(v);
@@ -265,6 +269,9 @@ std::string request_to_json(const Yue2Request * r, bool sparse) {
         yyjson_mut_obj_add_int(doc, root, "synth_batch_size", r->synth_batch_size);
     }
     add_sampling(doc, root, "abc_sampling", r->abc_sampling, d.abc_sampling, sparse);
+    if (!sparse || r->plan_retries != d.plan_retries) {
+        yyjson_mut_obj_add_int(doc, root, "plan_retries", r->plan_retries);
+    }
     add_sampling(doc, root, "semantic_sampling", r->semantic_sampling, d.semantic_sampling, sparse);
     if (!sparse || r->semantic_tokens != d.semantic_tokens) {
         yyjson_mut_obj_add_strncpy(doc, root, "semantic_tokens", r->semantic_tokens.c_str(), r->semantic_tokens.size());

src/request.h

@@ -53,6 +53,9 @@ struct Yue2Request {
 
     // sampling of each autoregressive stage, the checkpoint presets by default
     Yue2Sampling abc_sampling;
+    // a planned score nothing can be sung from (score-check.h) is planned
+    // again, song i's attempt k drawing with lm_seed + i + k * lm_batch_size
+    int plan_retries;  // 2, at most 8
     Yue2Sampling semantic_sampling;
 
     // semantic stream (CSV of codec values, 25 per second). Non empty replaces

src/score-check.h

@@ -0,0 +1,121 @@
+// score-check.h: a planned score nothing can be sung from
+//
+// Adapters pushed past their limits, or a seed that goes astray, make the
+// planner write garbage: no key, no meter, colons inside the note runs. The
+// semantic stage would then spend minutes on it. The plan is read once it is
+// written and the song stops there, in seconds, with what is wrong.
+#pragma once
+
+#include <cstdio>
+#include <string>
+
+// "M:4/4", "L:1/16": a positive fraction and nothing after it
+static bool yue2_score_fraction(const std::string & v) {
+    int  num = 0, den = 0;
+    char end = 0;
+    return sscanf(v.c_str(), "%d/%d%c", &num, &den, &end) == 2 && num > 0 && den > 0;
+}
+
+// A field line: a letter, then ':'
+static bool yue2_score_field(const std::string & line) {
+    return line.size() >= 2 && line[1] == ':' &&
+           ((line[0] >= 'A' && line[0] <= 'Z') || (line[0] >= 'a' && line[0] <= 'z'));
+}
+
+// The colons of a note line outside chord symbols, inline fields [K:...]
+// and repeat bars (|: :| ::)
+static int yue2_score_stray_colons(const std::string & line) {
+    int  stray  = 0;
+    bool quoted = false;
+    int  depth  = 0;
+    for (size_t i = 0; i < line.size(); i++) {
+        char c = line[i];
+        if (c == '"') {
+            quoted = !quoted;
+        } else if (quoted) {
+            continue;
+        } else if (c == '[') {
+            depth++;
+        } else if (c == ']') {
+            depth = depth > 0 ? depth - 1 : 0;
+        } else if (c == ':' && depth == 0) {
+            char before = i > 0 ? line[i - 1] : 0;
+            char after  = i + 1 < line.size() ? line[i + 1] : 0;
+            if (before != '|' && after != '|' && before != ':' && after != ':') {
+                stray++;
+            }
+        }
+    }
+    return stray;
+}
+
+// The start of a score for a log line: printable ASCII and newlines only, the
+// rest as '?'. A garbage plan holds partial UTF-8 sequences, and a reader of
+// the log as UTF-8 text would stop at the first one.
+static std::string yue2_score_excerpt(const std::string & abc, size_t limit) {
+    std::string out;
+    for (size_t i = 0; i < abc.size() && out.size() < limit; i++) {
+        unsigned char c = (unsigned char) abc[i];
+        out += (c == '\n' || (c >= 0x20 && c < 0x7F)) ? (char) c : '?';
+    }
+    return out;
+}
+
+// What makes a score unusable, "" when nothing does
+static std::string yue2_score_problem(const std::string & abc) {
+    bool        meter = false, length = false, key = false, voice = false, bars = false;
+    int         stray = 0;
+    size_t      from  = 0;
+    std::string first_stray;
+    while (from < abc.size()) {
+        size_t      nl   = abc.find('\n', from);
+        std::string line = abc.substr(from, nl == std::string::npos ? std::string::npos : nl - from);
+        from             = nl == std::string::npos ? abc.size() : nl + 1;
+        if (!line.empty() && line.back() == '\r') {
+            line.pop_back();
+        }
+        if (line.empty() || line[0] == '%') {
+            continue;
+        }
+        if (yue2_score_field(line)) {
+            std::string v = line.substr(2);
+            size_t      a = v.find_first_not_of(" \t");
+            v             = a == std::string::npos ? std::string() : v.substr(a);
+            while (!v.empty() && (v.back() == ' ' || v.back() == '\t')) {
+                v.pop_back();
+            }
+            if (!key && line[0] == 'M') {
+                meter = meter || yue2_score_fraction(v) || v == "C" || v == "C|" || v == "none";
+            } else if (!key && line[0] == 'L') {
+                length = length || yue2_score_fraction(v);
+            } else if (line[0] == 'K') {
+                key = key || !v.empty();
+            } else if (line[0] == 'V') {
+                voice = true;
+            }
+            continue;
+        }
+        bars = bars || line.find('|') != std::string::npos;
+        int here = yue2_score_stray_colons(line);
+        if (here > 0 && first_stray.empty()) {
+            first_stray = line.size() > 40 ? line.substr(0, 40) + "..." : line;
+        }
+        stray += here;
+    }
+    std::string missing;
+    for (auto [have, name] : { std::pair<bool, const char *>{ meter, "M:" }, { length, "L:" }, { key, "K:" } }) {
+        if (!have) {
+            missing += (missing.empty() ? "" : ", ") + std::string(name);
+        }
+    }
+    if (!missing.empty()) {
+        return "its header has no well-formed " + missing;
+    }
+    if (!voice || !bars) {
+        return "it has no voice with bars";
+    }
+    if (stray > 0) {
+        return "colons stand inside its note runs (" + std::to_string(stray) + ", first in \"" + first_stray + "\")";
+    }
+    return "";
+}

tests/CMakeLists.txt

@@ -35,3 +35,7 @@ link_ggml_backends(test-sheetsage)
 add_executable(test-tokenizer test-tokenizer.cpp)
 target_link_libraries(test-tokenizer PRIVATE yyjson)
 link_ggml_backends(test-tokenizer)
+
+# test-score-check: a planned score nothing can be sung from, pure CPU, no ggml.
+add_executable(test-score-check test-score-check.cpp)
+target_include_directories(test-score-check PRIVATE ${CMAKE_SOURCE_DIR}/src)

tests/test-score-check.cpp

@@ -0,0 +1,68 @@
+// test-score-check: a planned score nothing can be sung from, pure CPU.
+
+#include "score-check.h"
+
+#include <cstdio>
+#include <string>
+
+static int failures = 0;
+
+static void check(bool ok, const char * what) {
+    if (!ok) {
+        fprintf(stderr, "[ERROR] %s\n", what);
+        failures++;
+    }
+}
+
+static const std::string GOOD =
+    "X:1\nT:\nM:2/4\nL:1/32\nQ:1/4=70\nV: Vocal clef=treble name=\"Vocal Melody\" snm=\"Vocal\"\n"
+    "V: Ins clef=treble name=\"Ins Melody\" snm=\"Inst.\"\nK:Em\n% intro\nV: Vocal\n"
+    "z12\"Em\"z4|\"Em\"z16|\"C\"z16|\nV: Ins\nZ|E4B4f2g4d2-|d2B4B2A2G2F2G2|\n% verse\nV: Vocal\n"
+    "|:\"Em\"e8b8|\"C\"e8a4f4:|[K:G]\"G\"g16|\n";
+
+int main(int argc, char ** argv) {
+    // with files: the verdict on each score, for real plans
+    if (argc > 1) {
+        int bad = 0;
+        for (int i = 1; i < argc; i++) {
+            FILE * f = fopen(argv[i], "rb");
+            if (!f) {
+                fprintf(stderr, "[ERROR] cannot read %s\n", argv[i]);
+                return 1;
+            }
+            std::string text;
+            char        buf[4096];
+            size_t      n;
+            while ((n = fread(buf, 1, sizeof buf, f)) > 0) {
+                text.append(buf, n);
+            }
+            fclose(f);
+            std::string problem = yue2_score_problem(text);
+            if (!problem.empty()) {
+                bad++;
+                printf("%s: %s\n", argv[i], problem.c_str());
+            }
+        }
+        printf("%d of %d scores refused\n", bad, argc - 1);
+        return 0;
+    }
+    check(yue2_score_problem(GOOD).empty(), "a well-formed score passes");
+    check(yue2_score_problem("X:1\nM:C\nL:1/8\nK:C\nV:1\nCDEF|GABc|\n").empty(), "common time passes");
+    check(yue2_score_problem("X:1\nL:1/8\nK:C\nV:1\nCDEF|\n") == "its header has no well-formed M:", "no meter");
+    check(yue2_score_problem("X:1\nM:4/4\nL:1/8\nV:1\nCDEF|\n") == "its header has no well-formed K:", "no key");
+    check(yue2_score_problem("X:1\nM:4/x\nL:\nK:C\nV:1\nCDEF|\n") == "its header has no well-formed M:, L:",
+          "malformed meter and length");
+    check(yue2_score_problem("X:1\nM:4/4\nL:1/8\nK:C\n") == "it has no voice with bars", "no body");
+    check(yue2_score_problem("X:1\nM:4/4\nL:1/8\nK:C\nV:1\nCD:E:F|G2A2:B|\n").rfind("colons stand inside", 0) == 0,
+          "colons in a note run");
+    check(yue2_score_stray_colons("|:CDEF:|::GABc|") == 0, "repeat bars are no stray colons");
+    check(yue2_score_stray_colons("\"A:7\"C[K:G]D") == 0, "colons in chords and inline fields are not stray");
+    check(yue2_score_excerpt("K:C\n\xd0\xb0\xe4\xb8", 64) == "K:C\n????", "a log excerpt is printable ASCII");
+    check(yue2_score_excerpt("abcdef", 3) == "abc", "an excerpt stops at its limit");
+    if (failures) {
+        fprintf(stderr, "[ERROR] %d checks failed\n", failures);
+        return 1;
+    }
+    fprintf(stderr, "[OK] test-score-check\n");
+    return 0;
+}

tools/yue-plan.cpp

@@ -9,6 +9,7 @@
 #include "generate.h"
 #include "prompt.h"
 #include "request.h"
+#include "score-check.h"
 #include "version.h"
 
 #include <cstdio>
@@ -167,7 +168,11 @@ int main(int argc, char ** argv) {
     }
     const Yue2Generation & plan = plans[0];
 
-    std::string score = bpe_decode(&tok, plan.tokens);
+    std::string score   = bpe_decode(&tok, plan.tokens);
+    std::string problem = yue2_score_problem(score);
+    if (!problem.empty()) {
+        fprintf(stderr, "[Plan] WARNING: this score cannot be sung: %s\n", problem.c_str());
+    }
     if (!write_file(out_path, score)) {
         qw3lm_kv_free(&kv);
         qw3lm_free(&lm);

tools/yue-server.cpp

@@ -529,6 +529,11 @@ static bool validate(const std::string & json, httplib::Response & res, Yue2Requ
         res.set_content(json_string("error", "synth_batch_size must be between 1 and 9"), "application/json");
         return false;
     }
+    if (r->plan_retries < 0 || r->plan_retries > 8) {
+        res.status = 400;
+        res.set_content(json_string("error", "plan_retries must be between 0 and 8"), "application/json");
+        return false;
+    }
     if (!yue2_sampling_valid(r->abc_sampling, "abc") || !yue2_sampling_valid(r->semantic_sampling, "semantic")) {
         res.status = 400;
         res.set_content(json_string("error", "sampling preset outside the protocol bounds"), "application/json");