diff --git a/API.md b/API.md index 2cc189b..73a5d40 100644 --- a/API.md +++ b/API.md @@ -108,8 +108,8 @@ The following keys are available in context_settings: | Key | Type | Meaning | | ------------------------ | -------- | ---------------------------------------------------------------- | -| `context_size` | `number` | Equivalent to n_ctx = N and n_batch = N. | -| `n_ctx` | `number` | Text context length (tokens). `0` = from model. | +| `context_size` | `number` | Equivalent to `n_ctx = N` and `n_batch = N`. `0` = use the model's training window. | +| `n_ctx` | `number` | Text context length (tokens). `0` = use the model's training window. | | `n_batch` | `number` | **Logical** max batch size submitted to `llama_decode`. | | `n_ubatch` | `number` | **Physical** max micro-batch size. | | `n_seq_max` | `number` | Max concurrent sequences (parallel states for recurrent models). | @@ -199,7 +199,10 @@ SELECT llm_context_create_embedding(); **Description:** Creates a new inference context specifically set for chat conversation. -It is equivalent to `SELECT llm_context_create('context_size=4096');` +It applies no context settings of its own, so anything you do not pass is +llama.cpp's default — in particular the context length defaults to the model's +training window (`llm_model_n_ctx_train()`), not to a fixed size. Pass +`context_size` to bound it, and use `llm_context_size()` to confirm what you got. Context must explicitly created before performing any AI operation! @@ -220,7 +223,10 @@ SELECT llm_context_create_chat(); **Description:** Creates a new inference context specifically set for text generation. -It is equivalent to `SELECT llm_context_create('context_size=4096');` +It applies no context settings of its own, so anything you do not pass is +llama.cpp's default — in particular the context length defaults to the model's +training window (`llm_model_n_ctx_train()`), not to a fixed size. Pass +`context_size` to bound it, and use `llm_context_size()` to confirm what you got. Context must explicitly created before performing any AI operation! diff --git a/Package.swift b/Package.swift index ef05d6d..819f11c 100644 --- a/Package.swift +++ b/Package.swift @@ -14,8 +14,8 @@ let package = Package( targets: [ .binaryTarget( name: "aiBinary", - url: "https://github.com/sqliteai/sqlite-ai/releases/download/1.0.5/ai-apple-xcframework-1.0.5.zip", - checksum: "45b30694627d03bce68a5e79804822e61fce0b14ceb0723ecf7bd3a7dc59eb69" + url: "https://github.com/sqliteai/sqlite-ai/releases/download/1.0.6/ai-apple-xcframework-1.0.6.zip", + checksum: "64ecede326e1fb61f9cf47acea77f4aea5bef8b5ddd67aed40fa2ca184f604d7" ), .target( name: "ai", diff --git a/src/sqlite-ai.c b/src/sqlite-ai.c index a20e9b8..97d30b8 100644 --- a/src/sqlite-ai.c +++ b/src/sqlite-ai.c @@ -511,8 +511,8 @@ static bool llm_context_options_callback (void *ctx, void *xdata, const char *ke if (KEY_MATCHES(key, key_len, OPTION_KEY_CONTEXT_SIZE)) { int value = (int)strtol(buffer, NULL, 0); if (value >= 0) { - options->n_ctx = value; - options->n_batch = value; + options->n_ctx = value; // 0 = use the model's training window + if (value > 0) options->n_batch = value; // never drive n_batch to 0 } return true; } @@ -1821,9 +1821,13 @@ static bool llm_chat_generate_response (ai_context *ai, ai_cursor *c, bool *is_e llama_batch batch = ai->chat.batch; char *tok = ai->chat.token_text; - // check context space - uint32_t n_ctx = llama_n_ctx(ctx); - int32_t n_ctx_used = llama_memory_seq_pos_max(llama_get_memory(ctx), 0); + // check context space. seq_pos_max() is the highest position in the sequence, so + // occupancy is that + 1; treating it as a count let exactly one over-large batch + // through for llama_decode() to reject with "could not find a KV slot" instead. + // Both sides are signed: the comparison used to promote to unsigned, so an empty + // cache (seq_pos_max == -1) and a zero-token batch tripped the guard falsely. + int32_t n_ctx = (int32_t)llama_n_ctx(ctx); + int32_t n_ctx_used = llama_memory_seq_pos_max(llama_get_memory(ctx), 0) + 1; if (n_ctx_used + batch.n_tokens > n_ctx) { sqlite_common_set_error (ai->context, ai->vtab, SQLITE_ERROR, "Context size exceeded (%d, %d)", n_ctx, n_ctx_used + batch.n_tokens); return false; @@ -1990,7 +1994,17 @@ static bool llm_chat_run (ai_context *ai, ai_cursor *c, const char *user_prompt) // do not stream response and incrementally build the buffer bool is_eog = false; while (1) { - if (!llm_chat_generate_response (ai, NULL, &is_eog)) return false; + if (!llm_chat_generate_response (ai, NULL, &is_eog)) { + // The turn failed part-way. The user message is already in the history and + // its tokens are already in the KV cache, so commit whatever was generated + // before reporting: leaving the turn half-applied strands the user message + // with no assistant reply and leaves prev_len stale, which makes the delta + // for the *next* turn re-include it and fail too. The streaming cursor path + // already saves on close for the same reason. Errors from the save itself + // are allowed to replace the original message - they are worth reporting. + llm_chat_save_response(ai, messages, template); + return false; + } if (is_eog) break; } @@ -2724,6 +2738,14 @@ static void llm_context_free (sqlite3_context *context, int argc, sqlite3_value static bool llm_context_create_with_options (sqlite3_context *context, ai_context *ai, const char *options1, const char *options2) { struct llama_context_params ctx_params = llama_context_default_params(); + + // n_ctx = 0 tells llama.cpp to use the model's training window. Start from it so + // that "the caller did not ask for a context size" is an explicit sentinel rather + // than something inferred after the fact from the value: llama's own default is + // 512, so an explicit context_size=512 used to be indistinguishable from unset and + // was silently replaced by n_ctx_train. + ctx_params.n_ctx = 0; + if (parse_keyvalue_string(ai, options1, llm_context_options_callback, &ctx_params) == false) { sqlite_context_result_error(context, SQLITE_ERROR, "An error occurred while parsing options (%s)", options1); return false; @@ -2742,13 +2764,6 @@ static bool llm_context_create_with_options (sqlite3_context *context, ai_contex return false; } - // auto-size n_ctx to the model's training window when no explicit context_size was set - // llama_context_default_params() sets n_ctx=512; setting n_ctx=0 tells llama.cpp to use n_ctx_train - struct llama_context_params defaults = llama_context_default_params(); - if (ai->model && ctx_params.n_ctx == defaults.n_ctx) { - ctx_params.n_ctx = 0; - } - // for embedding contexts, clamp n_ctx to n_ctx_train to avoid position overflow if (ctx_params.embeddings && ai->model) { int n_ctx_train = llama_model_n_ctx_train(ai->model); diff --git a/src/sqlite-ai.h b/src/sqlite-ai.h index a710ed5..713d066 100644 --- a/src/sqlite-ai.h +++ b/src/sqlite-ai.h @@ -24,7 +24,7 @@ extern "C" { #endif -#define SQLITE_AI_VERSION "1.0.5" +#define SQLITE_AI_VERSION "1.0.6" SQLITE_AI_API int sqlite3_ai_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi); diff --git a/tests/c/unittest.c b/tests/c/unittest.c index 8868a35..a2362b5 100644 --- a/tests/c/unittest.c +++ b/tests/c/unittest.c @@ -159,6 +159,16 @@ static int install_seeded_chat_sampler(const test_env *env, sqlite3 *db) { return 0; } +// like exec_expect_ok, but hands back the error text so a test can compare two failures +static int exec_capture_error(const test_env *env, sqlite3 *db, const char *sql, char *err, size_t errlen) { + if (env->verbose) printf("[SQL] %s\n", sql); + char *errmsg = NULL; + int rc = sqlite3_exec(db, sql, NULL, NULL, &errmsg); + if (rc != SQLITE_OK && err && errlen) snprintf(err, errlen, "%s", errmsg ? errmsg : sqlite3_errmsg(db)); + if (errmsg) sqlite3_free(errmsg); + return (rc == SQLITE_OK) ? 0 : 1; +} + static int exec_select_rows(const test_env *env, sqlite3 *db, const char *sql, int *rows_out) { if (env->verbose) { printf("[SQL] %s\n", sql); @@ -607,6 +617,81 @@ static int test_llm_context_size_errors(const test_env *env) { return 1; } +// Regression: an explicit context_size=512 collided with the value +// llama_context_default_params() uses for n_ctx, so it was read as "the caller did +// not ask" and silently replaced by the model's training window - 32768 for the test +// model, 64x what was requested. +static int test_context_size_is_honoured(const test_env *env) { + sqlite3 *db = NULL; + if (open_db_and_load(env, &db) != SQLITE_OK) return 1; + + const char *model = env->model_path ? env->model_path : DEFAULT_MODEL_PATH; + char sqlbuf[512]; + snprintf(sqlbuf, sizeof(sqlbuf), "SELECT llm_model_load('%s');", model); + if (exec_expect_ok(env, db, sqlbuf) != 0) goto fail; + + int n_ctx_train = 0; + if (select_single_int(env, db, "SELECT llm_model_n_ctx_train();", &n_ctx_train) != 0) goto fail; + if (n_ctx_train <= 1024) { + fprintf(stderr, "[context_size_is_honoured] model trains at %d, too small to tell a " + "honoured context_size from an auto-sized one\n", n_ctx_train); + goto fail; + } + + // llama pads n_ctx up to a multiple of 256, so only exact multiples compare equal. + // 512 is the regression; the others guard against over-correcting. + const int sizes[] = {256, 512, 1024}; + for (size_t i = 0; i < sizeof(sizes) / sizeof(sizes[0]); ++i) { + snprintf(sqlbuf, sizeof(sqlbuf), "SELECT llm_context_create_chat('context_size=%d');", sizes[i]); + if (exec_expect_ok(env, db, sqlbuf) != 0) goto fail; + int n_ctx = 0; + if (select_single_int(env, db, "SELECT llm_context_size();", &n_ctx) != 0) goto fail; + if (env->verbose) printf("[context_size_is_honoured] asked %d, got %d\n", sizes[i], n_ctx); + if (n_ctx != sizes[i]) { + fprintf(stderr, "[context_size_is_honoured] context_size=%d produced n_ctx=%d\n", sizes[i], n_ctx); + goto fail; + } + if (exec_expect_ok(env, db, "SELECT llm_context_free();") != 0) goto fail; + } + + // n_ctx= is the other spelling of the same thing and hit the same collision + if (exec_expect_ok(env, db, "SELECT llm_context_create_chat('n_ctx=512');") != 0) goto fail; + int n_ctx = 0; + if (select_single_int(env, db, "SELECT llm_context_size();", &n_ctx) != 0) goto fail; + if (n_ctx != 512) { + fprintf(stderr, "[context_size_is_honoured] n_ctx=512 produced n_ctx=%d\n", n_ctx); + goto fail; + } + if (exec_expect_ok(env, db, "SELECT llm_context_free();") != 0) goto fail; + + // omitting it still auto-sizes to the model's training window + if (exec_expect_ok(env, db, "SELECT llm_context_create_chat();") != 0) goto fail; + if (select_single_int(env, db, "SELECT llm_context_size();", &n_ctx) != 0) goto fail; + if (n_ctx != n_ctx_train) { + fprintf(stderr, "[context_size_is_honoured] no context_size produced n_ctx=%d, expected %d\n", n_ctx, n_ctx_train); + goto fail; + } + if (exec_expect_ok(env, db, "SELECT llm_context_free();") != 0) goto fail; + + // and context_size=0 asks for that explicitly + if (exec_expect_ok(env, db, "SELECT llm_context_create_chat('context_size=0');") != 0) goto fail; + if (select_single_int(env, db, "SELECT llm_context_size();", &n_ctx) != 0) goto fail; + if (n_ctx != n_ctx_train) { + fprintf(stderr, "[context_size_is_honoured] context_size=0 produced n_ctx=%d, expected %d\n", n_ctx, n_ctx_train); + goto fail; + } + if (exec_expect_ok(env, db, "SELECT llm_context_free();") != 0) goto fail; + + if (exec_expect_ok(env, db, "SELECT llm_model_free();") != 0) goto fail; + + sqlite3_close_v2(db); + return assert_sqlite_memory_clean("context_size_is_honoured", env); + +fail: + if (db) sqlite3_close_v2(db); + return 1; +} + static int test_document_ingestion_flow(const test_env *env) { sqlite3 *db = NULL; if (open_db_and_load(env, &db) != SQLITE_OK) { @@ -1346,6 +1431,99 @@ static int test_chat_recreate_after_conversation(const test_env *env) { return 1; } +// Regression: running a chat out of context used to corrupt it. The turn bailed out of +// llm_chat_run() before llm_chat_save_response(), so the user message stayed in the +// history with no assistant reply and prev_len was never advanced - which made the next +// turn re-send the stranded message, so the reported requirement grew on every retry +// (266, 276, 286, 296 ...) and the chat could never be used again. The guard also +// treated llama_memory_seq_pos_max() (a position) as an occupancy count, so it let one +// over-large batch through for llama_decode() to reject instead. +static int test_chat_context_full_is_recoverable(const test_env *env) { + sqlite3 *db = NULL; + if (open_db_and_load(env, &db) != SQLITE_OK) return 1; + + const char *model = env->model_path ? env->model_path : DEFAULT_MODEL_PATH; + char sqlbuf[512]; + snprintf(sqlbuf, sizeof(sqlbuf), "SELECT llm_model_load('%s');", model); + if (exec_expect_ok(env, db, sqlbuf) != 0) goto fail; + if (install_seeded_chat_sampler(env, db) != 0) goto fail; + if (exec_expect_ok(env, db, "SELECT llm_context_create_chat('context_size=256');") != 0) goto fail; + if (exec_expect_ok(env, db, "SELECT llm_chat_create();") != 0) goto fail; + + // talk until the context runs out, then keep going: the failures are the point + const char *turn = "SELECT llm_chat_respond('Tell me about computers.');"; + #define MAX_FAILURES 4 + char errs[MAX_FAILURES][256] = {{0}}; + int ok_turns = 0, failed_turns = 0; + for (int i = 0; i < 40 && failed_turns < MAX_FAILURES; ++i) { + char err[256] = {0}; + if (exec_capture_error(env, db, turn, err, sizeof(err)) == 0) { + if (failed_turns == 0) ok_turns++; + continue; + } + snprintf(errs[failed_turns], sizeof(errs[0]), "%s", err); + failed_turns++; + } + + if (ok_turns == 0 || failed_turns < MAX_FAILURES) { + fprintf(stderr, "[chat_context_full_is_recoverable] wanted some good turns then %d failures, " + "got %d good and %d failed\n", MAX_FAILURES, ok_turns, failed_turns); + goto fail; + } + if (env->verbose) { + printf("[chat_context_full_is_recoverable] %d turns fit, then:\n", ok_turns); + for (int i = 0; i < failed_turns; ++i) printf(" #%d %s\n", i + 1, errs[i]); + } + + // the guard must catch it, not llama_decode() one token later + if (strstr(errs[0], "Context size exceeded") == NULL) { + fprintf(stderr, "[chat_context_full_is_recoverable] expected the context guard to report it, got: %s\n", errs[0]); + goto fail; + } + + // Once retrying, every attempt must report the SAME requirement: each one sends the + // same user message against the same full cache. A stranded user turn used to be + // re-sent on top, so the number climbed by a turn's worth of tokens every time. + // The FIRST failure is excluded on purpose - it can land mid-generation, where the + // batch is a single token, so it legitimately reports a different number from the + // prompt-batch failures that follow. + for (int i = 2; i < failed_turns; ++i) { + if (strcmp(errs[1], errs[i]) != 0) { + fprintf(stderr, "[chat_context_full_is_recoverable] failure grew across retries:\n"); + for (int j = 0; j < failed_turns; ++j) fprintf(stderr, " #%d %s\n", j + 1, errs[j]); + goto fail; + } + } + #undef MAX_FAILURES + + // and the history must not end on an orphaned user turn + if (exec_expect_ok(env, db, "SELECT llm_chat_save('context full');") != 0) goto fail; + ai_chat_message_row rows[128]; + int count = 0; + if (fetch_ai_chat_messages(env, db, rows, 128, &count) != 0) goto fail; + if (count < 2) { + fprintf(stderr, "[chat_context_full_is_recoverable] expected saved messages, got %d\n", count); + goto fail; + } + if (strcmp(rows[count - 1].role, "assistant") != 0) { + fprintf(stderr, "[chat_context_full_is_recoverable] history ends on a '%s' turn with no reply\n", + rows[count - 1].role); + goto fail; + } + + if (exec_expect_ok(env, db, "SELECT llm_chat_free();") != 0) goto fail; + if (exec_expect_ok(env, db, "SELECT llm_sampler_free();") != 0) goto fail; + if (exec_expect_ok(env, db, "SELECT llm_context_free();") != 0) goto fail; + if (exec_expect_ok(env, db, "SELECT llm_model_free();") != 0) goto fail; + + sqlite3_close_v2(db); + return assert_sqlite_memory_clean("chat_context_full_is_recoverable", env); + +fail: + if (db) sqlite3_close_v2(db); + return 1; +} + // Test chat vtab streaming produces tokens and saves response correctly across turns static int test_chat_vtab_multi_turn(const test_env *env) { sqlite3 *db = NULL; @@ -2093,6 +2271,7 @@ static const test_case TESTS[] = { {"llm_embed_generate_basic", test_llm_embed_generate_basic}, {"llm_embedding_then_chat", test_llm_embedding_then_chat}, {"llm_context_size_errors", test_llm_context_size_errors}, + {"context_size_is_honoured", test_context_size_is_honoured}, {"document_ingestion_flow", test_document_ingestion_flow}, {"llm_sampler_roundtrip", test_llm_sampler_roundtrip}, {"chat_default_sampler_autocreate", test_chat_default_sampler_autocreate}, @@ -2110,6 +2289,7 @@ static const test_case TESTS[] = { {"chat_system_prompt_after_first_response", test_chat_system_prompt_after_first_response}, {"chat_create_free_cycle", test_chat_create_free_cycle}, {"chat_recreate_after_conversation", test_chat_recreate_after_conversation}, + {"chat_context_full_is_recoverable", test_chat_context_full_is_recoverable}, {"chat_vtab_multi_turn", test_chat_vtab_multi_turn}, {"chat_save_restore_roundtrip", test_chat_save_restore_roundtrip}, {"chat_system_prompt_clear", test_chat_system_prompt_clear},