libs/vulkan-compute/src/vk_similarity.c

7#include "vk_similarity.h"

8#include "vk_compute.h"

9#include <stdlib.h>

10#include <string.h>

11#include <stdio.h>

12#include <pthread.h>

13#include <stdatomic.h>

14#include <time.h>

19struct VkSimilarityContext {

20 VkComputeContext* vk_ctx;

22 // Buffers

23 // Issue 9-002b: the sequential per-poem path was removed (the parallel

24 // single-dispatch path is the only one we run). Its scaffolding -- a

25 // per-source embedding buffer, a sequential output buffer, a CPU copy of

26 // the embeddings for source extraction, and a download scratch buffer --

27 // went with it, so all that remains is the one-time embedding upload and

28 // the full-matrix output.

29 VkComputeBuffer* embeddings_buffer; // All embeddings (GPU)

30 VkComputeBuffer* full_similarities_buffer; // Full triangular output (GPU) - parallel mode

32 // Pipelines

33 VkComputePipeline* similarity_full_pipeline; // Parallel full-matrix shader

35 // Metadata

36 uint32_t num_poems;

37 uint32_t embedding_dim;

38};

45VkSimilarityContext* vks_init(VkComputeContext* ctx,

46 const float* embeddings,

47 uint32_t num_poems,

48 uint32_t embedding_dim) {

49 if (!ctx || !embeddings || num_poems == 0 || embedding_dim == 0) {

50 fprintf(stderr, "[VKS ERROR] Invalid parameters to vks_init\n");

51 return NULL;

52 }

54 VkSimilarityContext* sim_ctx = (VkSimilarityContext*)calloc(1, sizeof(VkSimilarityContext));

55 if (!sim_ctx) {

56 fprintf(stderr, "[VKS ERROR] Failed to allocate similarity context\n");

57 return NULL;

58 }

60 sim_ctx->vk_ctx = ctx;

61 sim_ctx->num_poems = num_poems;

62 sim_ctx->embedding_dim = embedding_dim;

64 // The embeddings live on the GPU for the whole run. The parallel shader

65 // reads every source vector straight from this device buffer, so there is

66 // no per-poem re-upload and no host-side copy to keep around.

67 size_t embeddings_size = num_poems * embedding_dim * sizeof(float);

69 sim_ctx->embeddings_buffer = vkc_create_buffer(ctx, embeddings_size, VKC_BUFFER_DEVICE_LOCAL);

70 if (!sim_ctx->embeddings_buffer) {

71 fprintf(stderr, "[VKS ERROR] Failed to create GPU buffers\n");

72 vks_destroy(sim_ctx);

73 return NULL;

74 }

76 // Upload embeddings to GPU (one-time upload)

77 VkComputeResult result = vkc_upload_buffer(ctx, sim_ctx->embeddings_buffer,

78 (void*)embeddings, embeddings_size);

79 if (result != VKC_SUCCESS) {

80 fprintf(stderr, "[VKS ERROR] Failed to upload embeddings: %s\n",

81 vkc_get_error_string(result));

82 vks_destroy(sim_ctx);

83 return NULL;

84 }

86 return sim_ctx;

87}

94void vks_destroy(VkSimilarityContext* sim_ctx) {

95 if (!sim_ctx) return;

97 if (sim_ctx->similarity_full_pipeline) {

98 vkc_destroy_pipeline(sim_ctx->vk_ctx, sim_ctx->similarity_full_pipeline);

99 }

101 if (sim_ctx->full_similarities_buffer) {

102 vkc_destroy_buffer(sim_ctx->vk_ctx, sim_ctx->full_similarities_buffer);

103 }

105 if (sim_ctx->embeddings_buffer) {

106 vkc_destroy_buffer(sim_ctx->vk_ctx, sim_ctx->embeddings_buffer);

107 }

109 free(sim_ctx);

110}

127VkComputeResult vks_compute_all_similarities_parallel(

128 VkSimilarityContext* sim_ctx,

129 float* output_triangular) {

131 if (!sim_ctx || !output_triangular) {

132 return VKC_ERROR_COMMAND_EXECUTION_FAILED;

133 }

135 uint32_t num_poems = sim_ctx->num_poems;

136 uint64_t triangular_size = ((uint64_t)num_poems * (num_poems - 1)) / 2;

137 size_t buffer_size = triangular_size * sizeof(float);

139 printf("[VKS PARALLEL] Poems: %u, Pairs: %lu (%.1f MB)\n",

140 num_poems, (unsigned long)triangular_size, buffer_size / (1024.0 * 1024.0));

142 // Lazy-load the parallel shader if not already loaded

143 if (!sim_ctx->similarity_full_pipeline) {

144 printf("[VKS PARALLEL] Loading parallel similarity shader...\n");

145 sim_ctx->similarity_full_pipeline = vkc_create_pipeline(

146 sim_ctx->vk_ctx,

147 "libs/vulkan-compute/build/similarity_full.spv",

148 sizeof(uint32_t) * 2 // Push constants: num_poems, embedding_dim

149 );

151 if (!sim_ctx->similarity_full_pipeline) {

152 fprintf(stderr, "[VKS ERROR] Failed to load parallel similarity shader\n");

153 return VKC_ERROR_SHADER_LOAD_FAILED;

154 }

155 }

157 // Lazy-create the full output buffer if not already created

158 if (!sim_ctx->full_similarities_buffer) {

159 printf("[VKS PARALLEL] Allocating GPU output buffer (%.1f MB)...\n",

160 buffer_size / (1024.0 * 1024.0));

161 sim_ctx->full_similarities_buffer = vkc_create_buffer(

162 sim_ctx->vk_ctx, buffer_size, VKC_BUFFER_DEVICE_LOCAL

163 );

165 if (!sim_ctx->full_similarities_buffer) {

166 fprintf(stderr, "[VKS ERROR] Failed to create full similarities buffer\n");

167 return VKC_ERROR_BUFFER_CREATION_FAILED;

168 }

169 }

171 // Bind buffers to pipeline

172 VkComputeResult result;

174 result = vkc_bind_buffer(sim_ctx->vk_ctx, sim_ctx->similarity_full_pipeline,

175 0, sim_ctx->embeddings_buffer);

176 if (result != VKC_SUCCESS) {

177 fprintf(stderr, "[VKS ERROR] Failed to bind embeddings buffer\n");

178 return result;

179 }

181 result = vkc_bind_buffer(sim_ctx->vk_ctx, sim_ctx->similarity_full_pipeline,

182 1, sim_ctx->full_similarities_buffer);

183 if (result != VKC_SUCCESS) {

184 fprintf(stderr, "[VKS ERROR] Failed to bind output buffer\n");

185 return result;

186 }

188 // Prepare push constants

189 uint32_t push_constants[2] = {

190 num_poems,

191 sim_ctx->embedding_dim

192 };

194 // Calculate workgroup dispatch size

195 // Shader uses local_size_x=16, local_size_y=16

196 // Need to cover num_poems × num_poems grid (upper triangle computed)

197 uint32_t workgroups_x = (num_poems + 15) / 16;

198 uint32_t workgroups_y = (num_poems + 15) / 16;

200 printf("[VKS PARALLEL] Dispatching %u × %u = %u workgroups (256 threads each)\n",

201 workgroups_x, workgroups_y, workgroups_x * workgroups_y);

203 // Single dispatch for ALL pairs!

204 result = vkc_dispatch(sim_ctx->vk_ctx, sim_ctx->similarity_full_pipeline,

205 workgroups_x, workgroups_y, 1, push_constants);

206 if (result != VKC_SUCCESS) {

207 fprintf(stderr, "[VKS ERROR] Failed to dispatch parallel similarity shader\n");

208 return result;

209 }

211 // Download all results at once

212 result = vkc_download_buffer(sim_ctx->vk_ctx, sim_ctx->full_similarities_buffer,

213 output_triangular, buffer_size);

214 if (result != VKC_SUCCESS) {

215 fprintf(stderr, "[VKS ERROR] Failed to download similarity results\n");

216 return result;

217 }

219 return VKC_SUCCESS;

220}

228typedef struct {

229 uint32_t target_index; // poem_index of target

230 float similarity;

231} SimilarityPair;

234typedef struct {

235 // Shared (read-only after init)

236 const float* triangular_buffer;

237 uint32_t num_poems;

238 const uint32_t* poem_indices;

239 const char** poem_ids;

240 const char* output_dir;

242 // Atomic task counter (shared, written atomically)

243 atomic_uint* next_task;

245 // Per-thread statistics

246 uint32_t files_written;

247 uint32_t thread_id;

248} FileWriterContext;

251static int compare_similarity_desc(const void* a, const void* b) {

252 const SimilarityPair* pa = (const SimilarityPair*)a;

253 const SimilarityPair* pb = (const SimilarityPair*)b;

255 // Descending order (higher similarity first)

256 if (pa->similarity > pb->similarity) return -1;

257 if (pa->similarity < pb->similarity) return 1;

258 return 0;

259}

266static int write_similarity_file(

267 const float* triangular_buffer,

268 uint32_t num_poems,

269 const uint32_t* poem_indices,

270 const char** poem_ids,

271 const char* output_dir,

272 uint32_t array_index) {

274 uint32_t poem_index = poem_indices[array_index];

275 const char* poem_id = poem_ids[array_index];

277 // Calculate number of similarities for this poem

278 // For array_index i, we need similarities to all j where j > i (upper triangle)

279 // Plus we need to include similarities FROM earlier poems TO this one (lower triangle)

280 // Total: (num_poems - 1) similarities per poem

282 uint32_t total_sims = num_poems - 1;

283 if (total_sims == 0) {

284 // Single poem, create empty file

285 char filepath[512];

286 snprintf(filepath, sizeof(filepath), "%s/poem_index_%u.json", output_dir, poem_index);

288 FILE* f = fopen(filepath, "w");

289 if (!f) {

290 fprintf(stderr, "[VKS FILE ERROR] Failed to open: %s\n", filepath);

291 return -1;

292 }

294 fprintf(f, "{\n");

295 fprintf(f, " \"metadata\": {\n");

296 fprintf(f, " \"poem_id\": \"%s\",\n", poem_id);

297 fprintf(f, " \"poem_index\": %u,\n", poem_index);

298 fprintf(f, " \"total_comparisons\": 0,\n");

299 fprintf(f, " \"format\": \"triangular_upper\",\n");

300 fprintf(f, " \"method\": \"gpu_vulkan_parallel_c\"\n");

301 fprintf(f, " },\n");

302 fprintf(f, " \"similarities\": [],\n");

303 fprintf(f, " \"sorted_indices\": []\n");

304 fprintf(f, "}\n");

306 fclose(f);

307 return 0;

308 }

310 // Allocate pairs array

311 SimilarityPair* pairs = (SimilarityPair*)malloc(total_sims * sizeof(SimilarityPair));

312 if (!pairs) {

313 fprintf(stderr, "[VKS FILE ERROR] Failed to allocate pairs array\n");

314 return -1;

315 }

317 uint32_t pair_count = 0;

318 uint32_t i = array_index;

320 // Extract similarities from triangular buffer

321 // For each other poem j:

322 // If i < j: sim(i,j) is at triangular_index(i, j)

323 // If i > j: sim(i,j) = sim(j,i) at triangular_index(j, i)

324 for (uint32_t j = 0; j < num_poems; j++) {

325 if (j == i) continue; // Skip self

327 uint64_t tri_idx;

328 if (i < j) {

329 tri_idx = vks_triangular_index(i, j, num_poems);

330 } else {

331 tri_idx = vks_triangular_index(j, i, num_poems);

332 }

334 pairs[pair_count].target_index = poem_indices[j];

335 pairs[pair_count].similarity = triangular_buffer[tri_idx];

336 pair_count++;

337 }

339 // Sort by similarity (descending)

340 qsort(pairs, pair_count, sizeof(SimilarityPair), compare_similarity_desc);

342 // Write JSON file

343 char filepath[512];

344 snprintf(filepath, sizeof(filepath), "%s/poem_index_%u.json", output_dir, poem_index);

346 FILE* f = fopen(filepath, "w");

347 if (!f) {

348 fprintf(stderr, "[VKS FILE ERROR] Failed to open: %s\n", filepath);

349 free(pairs);

350 return -1;

351 }

353 // Write header

354 fprintf(f, "{\n");

355 fprintf(f, " \"metadata\": {\n");

356 fprintf(f, " \"poem_id\": \"%s\",\n", poem_id);

357 fprintf(f, " \"poem_index\": %u,\n", poem_index);

358 fprintf(f, " \"total_comparisons\": %u,\n", pair_count);

359 fprintf(f, " \"format\": \"full_bidirectional\",\n");

360 fprintf(f, " \"method\": \"gpu_vulkan_parallel_c\"\n");

361 fprintf(f, " },\n");

363 // Write similarities array

364 fprintf(f, " \"similarities\": [\n");

365 for (uint32_t k = 0; k < pair_count; k++) {

366 fprintf(f, " {\"id\": \"%u\", \"similarity\": %.8f}%s\n",

367 pairs[k].target_index,

368 pairs[k].similarity,

369 k < pair_count - 1 ? "," : "");

370 }

371 fprintf(f, " ],\n");

373 // Write sorted_indices array

374 fprintf(f, " \"sorted_indices\": [");

375 for (uint32_t k = 0; k < pair_count; k++) {

376 fprintf(f, "%u%s", pairs[k].target_index, k < pair_count - 1 ? ", " : "");

377 }

378 fprintf(f, "]\n");

380 fprintf(f, "}\n");

382 fclose(f);

383 free(pairs);

385 return 0;

386}

393static void* file_writer_thread(void* arg) {

394 FileWriterContext* ctx = (FileWriterContext*)arg;

396 while (1) {

397 // Atomically get next task

398 uint32_t task_idx = atomic_fetch_add(ctx->next_task, 1);

400 // Check if we're done

401 if (task_idx >= ctx->num_poems) {

402 break;

403 }

405 // Write file for this poem

406 int result = write_similarity_file(

407 ctx->triangular_buffer,

408 ctx->num_poems,

409 ctx->poem_indices,

410 ctx->poem_ids,

411 ctx->output_dir,

412 task_idx

413 );

415 if (result == 0) {

416 ctx->files_written++;

417 }

418 }

420 return NULL;

421}

428VkComputeResult vks_write_similarity_files_parallel(

429 const float* triangular_buffer,

430 uint32_t num_poems,

431 const uint32_t* poem_indices,

432 const char** poem_ids,

433 const char* output_dir,

434 uint32_t num_threads) {

436 if (!triangular_buffer || !poem_indices || !poem_ids || !output_dir) {

437 fprintf(stderr, "[VKS FILE ERROR] Invalid parameters\n");

438 return VKC_ERROR_COMMAND_EXECUTION_FAILED;

439 }

441 // Clamp thread count

442 if (num_threads < 1) num_threads = 1;

443 if (num_threads > 64) num_threads = 64;

445 printf("[VKS FILE] Writing %u similarity files with %u threads...\n", num_poems, num_threads);

447 time_t start_time = time(NULL);

449 // Create atomic task counter

450 atomic_uint next_task = ATOMIC_VAR_INIT(0);

452 // Allocate thread contexts

453 FileWriterContext* contexts = (FileWriterContext*)calloc(num_threads, sizeof(FileWriterContext));

454 pthread_t* threads = (pthread_t*)malloc(num_threads * sizeof(pthread_t));

456 if (!contexts || !threads) {

457 fprintf(stderr, "[VKS FILE ERROR] Failed to allocate thread resources\n");

458 free(contexts);

459 free(threads);

460 return VKC_ERROR_OUT_OF_MEMORY;

461 }

463 // Initialize contexts and spawn threads

464 for (uint32_t t = 0; t < num_threads; t++) {

465 contexts[t].triangular_buffer = triangular_buffer;

466 contexts[t].num_poems = num_poems;

467 contexts[t].poem_indices = poem_indices;

468 contexts[t].poem_ids = poem_ids;

469 contexts[t].output_dir = output_dir;

470 contexts[t].next_task = &next_task;

471 contexts[t].files_written = 0;

472 contexts[t].thread_id = t;

474 int rc = pthread_create(&threads[t], NULL, file_writer_thread, &contexts[t]);

475 if (rc != 0) {

476 fprintf(stderr, "[VKS FILE ERROR] Failed to create thread %u: %d\n", t, rc);

477 // Wait for already-created threads

478 for (uint32_t i = 0; i < t; i++) {

479 pthread_join(threads[i], NULL);

480 }

481 free(contexts);

482 free(threads);

483 return VKC_ERROR_COMMAND_EXECUTION_FAILED;

484 }

485 }

487 // Draw a live progress bar while the workers drain the task queue.

488 // next_task is the number of poems CLAIMED so far; the workers do all the

489 // file I/O, so this main thread is idle here and free to render. We poll

490 // the atomic rather than read each thread's files_written to avoid a data

491 // race on those counters. (vkc_progress_update clamps the brief overshoot

492 // that atomic_fetch_add produces as the last num_threads workers finish,

493 // and honours the TTY / --debug mode selection.)

494 struct timespec poll_interval = { 0, 100 * 1000 * 1000 }; // 100 ms

495 for (;;) {

496 uint32_t claimed = atomic_load(&next_task);

497 vkc_progress_update("[VKS FILE]", claimed, num_poems);

498 // Bar is full once every task is claimed; the final files may still be

499 // flushing, but the join below waits for them. Break to stop polling.

500 if (claimed >= num_poems) break;

501 nanosleep(&poll_interval, NULL);

502 }

503 vkc_progress_finish();

505 // Join threads (work is already done, so this returns promptly) and tally.

506 uint32_t total_written = 0;

507 for (uint32_t t = 0; t < num_threads; t++) {

508 pthread_join(threads[t], NULL);

509 total_written += contexts[t].files_written;

510 }

512 time_t end_time = time(NULL);

513 double elapsed = difftime(end_time, start_time);

514 double rate = elapsed > 0 ? total_written / elapsed : total_written;

516 printf("[VKS FILE] ✅ Wrote %u files in %.0f seconds (%.1f files/sec)\n",

517 total_written, elapsed, rate);

519 free(contexts);

520 free(threads);

522 return VKC_SUCCESS;

523}

531typedef struct {

532 uint32_t* sorted_indices; // Array of poem_indices sorted by similarity

533 uint32_t count; // Number of entries (num_poems - 1)

534} PoemRankings;

537typedef struct {

538 // Shared (read-only)

539 const float* triangular_buffer;

540 uint32_t num_poems;

541 const uint32_t* poem_indices;

542 uint32_t top_k; // keep only the top-K neighbours per poem (0 = keep all)

544 // Shared output (written by threads, each to its own slot)

545 PoemRankings* all_rankings;

547 // Atomic task counter

548 atomic_uint* next_task;

550 // Per-thread stats

551 uint32_t poems_sorted;

552 uint32_t thread_id;

553} CacheGenContext;

560static void* cache_gen_thread(void* arg) {

561 CacheGenContext* ctx = (CacheGenContext*)arg;

563 uint32_t total_sims = ctx->num_poems - 1;

565 // Allocate thread-local pairs array (reused for each poem)

566 SimilarityPair* pairs = (SimilarityPair*)malloc(total_sims * sizeof(SimilarityPair));

567 if (!pairs) {

568 fprintf(stderr, "[VKS CACHE ERROR] Thread %u failed to allocate pairs\n", ctx->thread_id);

569 return NULL;

570 }

572 while (1) {

573 // Atomically get next task (array index to process)

574 uint32_t array_idx = atomic_fetch_add(ctx->next_task, 1);

576 if (array_idx >= ctx->num_poems) {

577 break;

578 }

580 // Extract and sort similarities for this poem

581 uint32_t pair_count = 0;

583 for (uint32_t j = 0; j < ctx->num_poems; j++) {

584 if (j == array_idx) continue;

586 uint64_t tri_idx;

587 if (array_idx < j) {

588 tri_idx = vks_triangular_index(array_idx, j, ctx->num_poems);

589 } else {

590 tri_idx = vks_triangular_index(j, array_idx, ctx->num_poems);

591 }

593 pairs[pair_count].target_index = ctx->poem_indices[j];

594 pairs[pair_count].similarity = ctx->triangular_buffer[tri_idx];

595 pair_count++;

596 }

598 // Sort by similarity (descending)

599 qsort(pairs, pair_count, sizeof(SimilarityPair), compare_similarity_desc);

601 // Keep only the top-K nearest neighbours. The list is already sorted

602 // descending, so the top-K are simply pairs[0..K-1]. top_k == 0 means keep

603 // all (backward compatible). This is THE memory cap (Issue 10-057): every

604 // place this list later lives -- this RAM array, the JSON written to disk,

605 // and the Lua table the HTML stage parses it back into -- shrinks by the

606 // same factor, because they are all this same data at different moments.

607 uint32_t keep = pair_count;

608 if (ctx->top_k > 0 && ctx->top_k < keep) keep = ctx->top_k;

610 // Allocate and fill the (capped) sorted indices for this poem

611 ctx->all_rankings[array_idx].sorted_indices = (uint32_t*)malloc(keep * sizeof(uint32_t));

612 ctx->all_rankings[array_idx].count = keep;

614 if (ctx->all_rankings[array_idx].sorted_indices) {

615 for (uint32_t k = 0; k < keep; k++) {

616 ctx->all_rankings[array_idx].sorted_indices[k] = pairs[k].target_index;

617 }

618 ctx->poems_sorted++;

619 }

620 }

622 free(pairs);

623 return NULL;

624}

631VkComputeResult vks_write_rankings_cache_parallel(

632 const float* triangular_buffer,

633 uint32_t num_poems,

634 const uint32_t* poem_indices,

635 const char* cache_file,

636 uint32_t num_threads,

637 uint32_t top_k) {

639 if (!triangular_buffer || !poem_indices || !cache_file) {

640 fprintf(stderr, "[VKS CACHE ERROR] Invalid parameters\n");

641 return VKC_ERROR_COMMAND_EXECUTION_FAILED;

642 }

644 if (num_threads < 1) num_threads = 1;

645 if (num_threads > 64) num_threads = 64;

647 printf("[VKS CACHE] Generating rankings cache with %u threads...\n", num_threads);

648 time_t start_time = time(NULL);

650 // Allocate rankings array (one entry per poem)

651 PoemRankings* all_rankings = (PoemRankings*)calloc(num_poems, sizeof(PoemRankings));

652 if (!all_rankings) {

653 fprintf(stderr, "[VKS CACHE ERROR] Failed to allocate rankings array\n");

654 return VKC_ERROR_OUT_OF_MEMORY;

655 }

657 // Create atomic task counter

658 atomic_uint next_task = ATOMIC_VAR_INIT(0);

660 // Allocate thread contexts

661 CacheGenContext* contexts = (CacheGenContext*)calloc(num_threads, sizeof(CacheGenContext));

662 pthread_t* threads = (pthread_t*)malloc(num_threads * sizeof(pthread_t));

664 if (!contexts || !threads) {

665 fprintf(stderr, "[VKS CACHE ERROR] Failed to allocate thread resources\n");

666 free(all_rankings);

667 free(contexts);

668 free(threads);

669 return VKC_ERROR_OUT_OF_MEMORY;

670 }

672 // Initialize and spawn threads

673 for (uint32_t t = 0; t < num_threads; t++) {

674 contexts[t].triangular_buffer = triangular_buffer;

675 contexts[t].num_poems = num_poems;

676 contexts[t].poem_indices = poem_indices;

677 contexts[t].top_k = top_k;

678 contexts[t].all_rankings = all_rankings;

679 contexts[t].next_task = &next_task;

680 contexts[t].poems_sorted = 0;

681 contexts[t].thread_id = t;

683 int rc = pthread_create(&threads[t], NULL, cache_gen_thread, &contexts[t]);

684 if (rc != 0) {

685 fprintf(stderr, "[VKS CACHE ERROR] Failed to create thread %u\n", t);

686 for (uint32_t i = 0; i < t; i++) {

687 pthread_join(threads[i], NULL);

688 }

689 free(all_rankings);

690 free(contexts);

691 free(threads);

692 return VKC_ERROR_COMMAND_EXECUTION_FAILED;

693 }

694 }

696 // Wait for all threads

697 uint32_t total_sorted = 0;

698 for (uint32_t t = 0; t < num_threads; t++) {

699 pthread_join(threads[t], NULL);

700 total_sorted += contexts[t].poems_sorted;

701 }

703 time_t sort_time = time(NULL);

704 printf("[VKS CACHE] Parallel sorting complete: %u poems in %ld seconds\n",

705 total_sorted, (long)(sort_time - start_time));

707 // Write cache file

708 printf("[VKS CACHE] Writing cache file: %s\n", cache_file);

710 FILE* f = fopen(cache_file, "w");

711 if (!f) {

712 fprintf(stderr, "[VKS CACHE ERROR] Failed to open cache file: %s\n", cache_file);

713 // Cleanup

714 for (uint32_t i = 0; i < num_poems; i++) {

715 free(all_rankings[i].sorted_indices);

716 }

717 free(all_rankings);

718 free(contexts);

719 free(threads);

720 return VKC_ERROR_COMMAND_EXECUTION_FAILED;

721 }

723 // Write JSON header

724 fprintf(f, "{\n");

725 fprintf(f, " \"metadata\": {\n");

726 fprintf(f, " \"total_poems\": %u,\n", num_poems);

727 fprintf(f, " \"algorithm\": \"gpu_vulkan_parallel_c\",\n");

728 fprintf(f, " \"format\": \"pre_sorted_rankings\",\n");

729 fprintf(f, " \"sort_threads\": %u,\n", num_threads);

730 fprintf(f, " \"top_k\": %u,\n", top_k);

731 fprintf(f, " \"description\": \"Pre-sorted similarity rankings, top-K neighbours per poem (top_k=0 means all)\"\n");

732 fprintf(f, " },\n");

734 // Write rankings. This is a single-threaded serialization of every poem's

735 // sorted neighbour list (~num_poems² integers as text), so it is the slow

736 // part of cache writing -- worth a live progress bar. We update every 64

737 // poems (and on the final one) to keep the bar smooth without paying a

738 // render call per iteration.

739 fprintf(f, " \"rankings\": {\n");

741 for (uint32_t i = 0; i < num_poems; i++) {

742 uint32_t poem_index = poem_indices[i];

743 PoemRankings* rankings = &all_rankings[i];

745 fprintf(f, " \"%u\": [", poem_index);

747 if (rankings->sorted_indices && rankings->count > 0) {

748 for (uint32_t k = 0; k < rankings->count; k++) {

749 fprintf(f, "%u", rankings->sorted_indices[k]);

750 if (k < rankings->count - 1) {

751 fprintf(f, ", ");

752 }

753 }

754 }

756 fprintf(f, "]%s\n", i < num_poems - 1 ? "," : "");

758 if ((i % 64) == 0 || i == num_poems - 1) {

759 vkc_progress_update("[VKS CACHE]", i + 1, num_poems);

760 }

761 }

762 vkc_progress_finish();

764 fprintf(f, " }\n");

765 fprintf(f, "}\n");

767 fclose(f);

769 // Cleanup

770 for (uint32_t i = 0; i < num_poems; i++) {

771 free(all_rankings[i].sorted_indices);

772 }

773 free(all_rankings);

774 free(contexts);

775 free(threads);

777 time_t end_time = time(NULL);

778 printf("[VKS CACHE] ✅ Cache written in %ld seconds total\n", (long)(end_time - start_time));

780 return VKC_SUCCESS;

781}