From 2186d96fccfc525884f1b3fb722c40642891ab0a Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fran=C3=A7ois=20Fleuret?= Date: Wed, 26 Jun 2024 08:41:45 +0200 Subject: [PATCH] Update. --- main.py | 60 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/main.py b/main.py index c5acea7..d0de5af 100755 --- a/main.py +++ b/main.py @@ -355,12 +355,19 @@ def create_c_quizzes( nb_for_test=100, min_ave_seq_logproba=None, ): - kept = [] + # We will store the generated quizzes for each number of + # correct prediction + recorded = dict([(n, []) for n in range(len(models) + 1)]) + model_indexes = [] sum_logits, sum_nb_c_quizzes = 0, 0 + nb_correct_to_validate = len(models) - 1 - while sum([x.size(0) for x in kept]) < nb_for_train + nb_for_test: - nb_to_generate = nb_for_train + nb_for_test + while ( + sum([x.size(0) for x in recorded[nb_correct_to_validate]]) + < nb_for_train + nb_for_test + ): + nb_to_validate = nb_for_train + nb_for_test if len(model_indexes) == 0: model_indexes = [i.item() for i in torch.randperm(len(models))] @@ -368,7 +375,7 @@ def create_c_quizzes( model = models[model_indexes.pop()] new_c_quizzes, nb_correct, ave_seq_logproba = quizz_machine.create_c_quizzes( - nb=nb_to_generate, + nb=nb_to_validate, model_for_generation=model, models_for_validation=models, min_ave_seq_logproba=min_ave_seq_logproba, @@ -380,33 +387,44 @@ def create_c_quizzes( sum_logits += new_c_quizzes.size(0) * ave_seq_logproba sum_nb_c_quizzes += new_c_quizzes.size(0) - to_keep = new_c_quizzes[nb_correct == len(models) - 1] - if args.dirty_debug: - to_keep = new_c_quizzes[ - torch.randint(3, (new_c_quizzes.size(0),), device=new_c_quizzes.device) - == 0 - ] + nb_correct = torch.randint( + len(models) + 1, nb_correct.size(), device=new_c_quizzes.device + ) + + for n in range(nb_correct.max() + 1): + recorded[n].append(new_c_quizzes[nb_correct == n].clone()) - kept.append(to_keep) + nb_validated = sum([x.size(0) for x in recorded[nb_correct_to_validate]]) + nb_generated = sum( + [sum([x.size(0) for x in recorded[n]]) for n in recorded.keys()] + ) log_string( - f"keep c_quizzes {to_keep.size(0)}/{new_c_quizzes.size(0)} ({to_keep.size(0)*100/new_c_quizzes.size(0):.02f}%) total {sum([ x.size(0) for x in kept])}/{nb_to_generate}" + f"keep c_quizzes {nb_validated*100/nb_generated:.02f}% kept total {nb_validated}/{nb_to_validate}" ) - new_c_quizzes = torch.cat(kept, dim=0) - new_c_quizzes = new_c_quizzes[ - torch.randperm(new_c_quizzes.size(0))[: nb_for_train + nb_for_test] - ] + # concatenate and shuffle + for n in recorded.keys(): + if len(recorded[n]) > 0: + q = torch.cat(recorded[n], dim=0) + q = q[torch.randperm(q.size(0), device=q.device)] + recorded[n] = q + else: + del recorded[n] + + new_c_quizzes = recorded[nb_correct_to_validate][: nb_for_train + nb_for_test] quizz_machine.store_c_quizzes(new_c_quizzes[:nb_for_train], for_train=True) quizz_machine.store_c_quizzes(new_c_quizzes[nb_for_train:], for_train=False) - quizz_machine.problem.save_quizzes( - new_c_quizzes[:72], - args.result_dir, - f"culture_c_quiz_{n_epoch:04d}_{model.id:02d}", - ) + for n in recorded.keys(): + s = "_validated" if n == nb_correct_to_validate else "" + quizz_machine.problem.save_quizzes( + recorded[n][:72], + args.result_dir, + f"culture_c_quiz_{n_epoch:04d}_N{n}{s}", + ) return sum_logits / sum_nb_c_quizzes -- 2.39.5