From 981341455b9bfc96439edb007394ff834e399daf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albin=20Ahlb=C3=A4ck?= Date: Tue, 14 Nov 2023 23:36:30 +0100 Subject: [PATCH 1/4] Use binary represention for relations in qsieve --- doc/source/qsieve.rst | 19 ++-- src/qsieve.h | 4 +- src/qsieve/large_prime_variant.c | 176 ++++++++++++++++++++++--------- 3 files changed, 140 insertions(+), 59 deletions(-) diff --git a/doc/source/qsieve.rst b/doc/source/qsieve.rst index 659354226e..0543a0ce61 100644 --- a/doc/source/qsieve.rst +++ b/doc/source/qsieve.rst @@ -91,13 +91,18 @@ Call for initialization of polynomial, sieving, and scanning of sieve for all the possible polynomials for particular hypercube i.e. `A`. -.. function:: void qsieve_write_to_file(qs_t qs_inf, mp_limb_t prime, fmpz_t Y, qs_poly_t poly) - - Write a relation to the file. Format is as follows, - first write large prime, in case of full relation it is 1, then write exponent - of small primes, then write number of factor followed by offset of factor in - factor base and their exponent and at last value of `Q(x)` for particular relation. - each relation is written in new line. +.. function:: void qsieve_write_to_file(qs_t qs_inf, mp_limb_t prime, const fmpz_t Y, const qs_poly_t poly) + + Write a relation to the file in a binary format as follows. First, write + large prime of size ``sizeof(mp_limb_t)``, in case of full relation it is 1. + After this, write the number of small primes with size ``sizeof(slong)``. + Then, write the small primes, with a total size of + ``number_of_small_primes * sizeof(slong)``. Then, write the number of + factors with a size of ``sizeof(slong)``. After that, write the factors and + their exponents in the format ``factor_1, exponent_1, factor_2, ...``, all + with a total size of ``2 * number_of_factors * sizeof(slong)``. Then write + ``Y`` with the size of ``Y`` first (size ``sizeof(slong)``, that may be + negative), and then its limbs (size ``Y_size * sizeof(mp_limb_t)``). .. function:: hash_t * qsieve_get_table_entry(qs_t qs_inf, mp_limb_t prime) diff --git a/src/qsieve.h b/src/qsieve.h index 5f557a4421..ab4377e1bd 100644 --- a/src/qsieve.h +++ b/src/qsieve.h @@ -351,13 +351,13 @@ int qsieve_relations_cmp(const void * a, const void * b); slong qsieve_merge_relations(qs_t qs_inf); void qsieve_write_to_file(qs_t qs_inf, mp_limb_t prime, - fmpz_t Y, qs_poly_t poly); + const fmpz_t Y, const qs_poly_t poly); hash_t * qsieve_get_table_entry(qs_t qs_inf, mp_limb_t prime); void qsieve_add_to_hashtable(qs_t qs_inf, mp_limb_t prime); -relation_t qsieve_parse_relation(qs_t qs_inf, char * str); +relation_t qsieve_parse_relation(qs_t qs_inf); relation_t qsieve_merge_relation(qs_t qs_inf, relation_t a, relation_t b); diff --git a/src/qsieve/large_prime_variant.c b/src/qsieve/large_prime_variant.c index a8f4849f6d..3b93d3ee3c 100644 --- a/src/qsieve/large_prime_variant.c +++ b/src/qsieve/large_prime_variant.c @@ -99,29 +99,74 @@ int qsieve_is_relation(qs_t qs_inf, relation_t a) /* Write partial or full relation to file -*/ -void qsieve_write_to_file(qs_t qs_inf, mp_limb_t prime, fmpz_t Y, qs_poly_t poly) + + The layout is as follows: + total write size of relation (including this write size) + large prime (1 * mp_limb_t) + number of small primes (1 * slong) + small primes (number of small primes * slong) + number of factors (1 * slong) + (factor, exponent) (number of factors * fac_t) + Y->_mp_size (1 * slong) + Y->_mp_d (Y->_mp_size * mp_limb_t) + */ +void qsieve_write_to_file(qs_t qs_inf, mp_limb_t prime, const fmpz_t Y, const qs_poly_t poly) { - slong i; - char * str = NULL; slong num_factors = poly->num_factors; slong * small = poly->small; fac_t * factor = poly->factor; + slong Ysz; + slong write_size; + + /* Get size of Y */ + Ysz = COEFF_IS_MPZ(*Y) ? COEFF_TO_PTR(*Y)->_mp_size : FLINT_SGN(*Y); + + /* Write size of relation */ + write_size = + sizeof(slong) /* total write size */ + + sizeof(mp_limb_t) /* large prime */ + + sizeof(slong) /* number of small primes */ + + sizeof(slong) * qs_inf->small_primes /* small primes */ + + sizeof(slong) /* number of factors */ + + sizeof(fac_t) * num_factors /* factors */ + + sizeof(slong) /* Y->_mp_size */ + + sizeof(mp_limb_t) * (Ysz != 0 ? FLINT_ABS(Ysz) : 1); /* Y->_mp_d */ + fwrite(&write_size, sizeof(slong), 1, (FILE *) qs_inf->siqs); - flint_fprintf((FILE *) qs_inf->siqs, "%X ", prime); /* write large prime */ + /* Write large prime */ + fwrite(&prime, sizeof(mp_limb_t), 1, (FILE *) qs_inf->siqs); - for (i = 0; i < qs_inf->small_primes; i++) /* write small primes */ - flint_fprintf((FILE *) qs_inf->siqs, "%X ", small[i]); + /* NOTE: We do not have to write small primes. */ + /* Write number of small primes */ + fwrite(&qs_inf->small_primes, sizeof(slong), 1, (FILE *) qs_inf->siqs); - flint_fprintf((FILE *) qs_inf->siqs, "%X ", num_factors); /* write number of factors */ + /* Write small primes */ + fwrite(small, sizeof(slong), qs_inf->small_primes, (FILE *) qs_inf->siqs); - for (i = 0; i < num_factors; i++) /* write factor along with exponent */ - flint_fprintf((FILE *) qs_inf->siqs, "%wx %X ", factor[i].ind, factor[i].exp); + /* Write number of factors */ + fwrite(&num_factors, sizeof(slong), 1, (FILE *) qs_inf->siqs); - str = fmpz_get_str(str, 16, Y); /* converting value of 'Y' to hex */ + /* Write factors and exponents */ + fwrite(factor, sizeof(fac_t), num_factors, (FILE *) qs_inf->siqs); - flint_fprintf((FILE *) qs_inf->siqs, "%s\n", str); /* write value of 'Y' */ - flint_free(str); + /* Write Y->_mp_size (or mock it) */ + fwrite(&Ysz, sizeof(slong), 1, (FILE *) qs_inf->siqs); + + /* Write Y->_mp_d (or mock it) */ + if (!COEFF_IS_MPZ(*Y)) + { + slong abslimb = FLINT_ABS(*Y); + + /* Write mock Y->_mp_d */ + fwrite(&abslimb, sizeof(mp_limb_t), 1, (FILE *) qs_inf->siqs); + } + else + { + mp_srcptr Yd = COEFF_TO_PTR(*Y)->_mp_d; + + /* Write Y->_mp_d */ + fwrite(Yd, sizeof(mp_limb_t), FLINT_ABS(Ysz), (FILE *) qs_inf->siqs); + } } /****************************************************************************** @@ -206,52 +251,71 @@ void qsieve_add_to_hashtable(qs_t qs_inf, mp_limb_t prime) given a string representing a relation, parse it to obtain relation */ -relation_t qsieve_parse_relation(qs_t qs_inf, char * str) +relation_t qsieve_parse_relation(qs_t qs_inf) { - slong i; - char * next; relation_t rel; + slong Ysz; + /* NOTE: write_size and large prime is already read in + * qsieve_process_relation. */ + + /* Get large prime (is always one) */ rel.lp = UWORD(1); - rel.small = flint_malloc(qs_inf->small_primes * sizeof(slong)); - rel.factor = flint_malloc(qs_inf->max_factors * sizeof(fac_t)); - for (i = 0; i < qs_inf->small_primes; i++) - { - while (isspace(*str)) - str++; + /* NOTE: We can use qs_inf->small_primes here instead of reading. */ + /* Get number of small primes */ + fread(&rel.small_primes, sizeof(slong), 1, (FILE *) qs_inf->siqs); - rel.small[i] = strtoul(str, &next, 16); - str = next; - } + /* Get small primes */ + rel.small = flint_malloc(rel.small_primes * sizeof(slong)); + fread(rel.small, sizeof(slong), rel.small_primes, (FILE *) qs_inf->siqs); - while (isspace(*str)) - str++; + /* Get number of factors */ + fread(&rel.num_factors, sizeof(slong), 1, (FILE *) qs_inf->siqs); - rel.num_factors = strtoul(str, &next, 16); - rel.small_primes = qs_inf->small_primes; - str = next; + /* Get factors */ + rel.factor = flint_malloc(rel.num_factors * sizeof(fac_t)); + fread(rel.factor, sizeof(fac_t), rel.num_factors, (FILE *) qs_inf->siqs); - for (i = 0; i < rel.num_factors; i++) - { - while (isspace(*str)) - str++; + /* Get Ysz */ + Ysz = 0; + fread(&Ysz, sizeof(slong), 1, (FILE *) qs_inf->siqs); - rel.factor[i].ind = strtoul(str, &next, 16); - str = next; + /* Get Y */ + fmpz_init(rel.Y); + if (FLINT_ABS(Ysz) <= 1) + { + mp_limb_t abslimb = 0; - while (isspace(*str)) - str++; + fread(&abslimb, sizeof(mp_limb_t), 1, (FILE *) qs_inf->siqs); - rel.factor[i].exp = strtoul(str, &next, 16); - str = next; +#if COEFF_MAX != -COEFF_MIN +# error +#endif + if (abslimb <= (mp_limb_t) COEFF_MAX) + { + *rel.Y = (Ysz < 0) ? -abslimb : abslimb; + } + else + { + mpz_ptr mY = _fmpz_new_mpz(); + mY->_mp_size = Ysz; + *mY->_mp_d = abslimb; + *rel.Y = PTR_TO_COEFF(mY); + } } + else + { + mpz_ptr mY = _fmpz_new_mpz(); - while (isspace(*str)) - str++; + mY->_mp_size = Ysz; - fmpz_init(rel.Y); - fmpz_set_str(rel.Y, str, 16); + if (mY->_mp_alloc < FLINT_ABS(Ysz)) + _mpz_realloc(mY, FLINT_ABS(Ysz)); + + fread(mY->_mp_d, sizeof(mp_limb_t), FLINT_ABS(Ysz), (FILE *) qs_inf->siqs); + *rel.Y = PTR_TO_COEFF(mY); + } return rel; } @@ -490,8 +554,6 @@ void qsieve_insert_relation(qs_t qs_inf, relation_t * rel_list, slong num_relati */ int qsieve_process_relation(qs_t qs_inf) { - char buf[1024]; - char * str; slong i, num_relations = 0, num_relations2; slong rel_list_length; slong rlist_length; @@ -513,23 +575,37 @@ int qsieve_process_relation(qs_t qs_inf) printf("Getting relations\n"); #endif - while (fgets(buf, sizeof(buf), (FILE *) qs_inf->siqs) != NULL) + while (1) { - prime = strtoul(buf, &str, 16); + int siqs_eof; + slong write_size = 0; + + siqs_eof = !fread(&write_size, sizeof(slong), 1, (FILE *) qs_inf->siqs); + + if (siqs_eof) + break; + + fread(&prime, sizeof(mp_limb_t), 1, (FILE *) qs_inf->siqs); entry = qsieve_get_table_entry(qs_inf, prime); if (num_relations == rel_size) { - rel_list = (relation_t *) flint_realloc(rel_list, 2*rel_size * sizeof(relation_t)); + rel_list = (relation_t *) flint_realloc(rel_list, 2 * rel_size * sizeof(relation_t)); rel_size *= 2; } if (prime == 1 || entry->count >= 2) { - rel_list[num_relations] = qsieve_parse_relation(qs_inf, str); + rel_list[num_relations] = qsieve_parse_relation(qs_inf); rel_list[num_relations].lp = prime; num_relations++; } + else + { + /* We have to get to the next relation in the file. We have already + * read write_size (is a slong) and large prime (is an mp_limb_t).*/ + fseek((FILE *) qs_inf->siqs, write_size - sizeof(slong) - sizeof(mp_limb_t), SEEK_CUR); + } } if(fclose((FILE *) qs_inf->siqs)) From 0bb73e1d8939873f2c4415a2b0a183ba231851ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albin=20Ahlb=C3=A4ck?= Date: Wed, 15 Nov 2023 13:21:41 +0100 Subject: [PATCH 2/4] Open qsieve tmp file in binary mode --- src/qsieve/factor.c | 6 +++--- src/qsieve/large_prime_variant.c | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/qsieve/factor.c b/src/qsieve/factor.c index 626526887c..dd75a71e67 100644 --- a/src/qsieve/factor.c +++ b/src/qsieve/factor.c @@ -226,7 +226,7 @@ void qsieve_factor(fmpz_factor_t factors, const fmpz_t n) flint_printf("Exception (qsieve_factor). GetTempFileNameA() failed.\n"); flint_abort(); } - qs_inf->siqs = (FLINT_FILE *) fopen(qs_inf->fname, "w"); + qs_inf->siqs = (FLINT_FILE *) fopen(qs_inf->fname, "wb"); if (qs_inf->siqs == NULL) flint_throw(FLINT_ERROR, "fopen failed\n"); #else @@ -235,7 +235,7 @@ void qsieve_factor(fmpz_factor_t factors, const fmpz_t n) if (fd == -1) flint_throw(FLINT_ERROR, "mkstemp failed\n"); - qs_inf->siqs = (FLINT_FILE *) fdopen(fd, "w"); + qs_inf->siqs = (FLINT_FILE *) fdopen(fd, "wb"); if (qs_inf->siqs == NULL) flint_throw(FLINT_ERROR, "fdopen failed\n"); #endif @@ -406,7 +406,7 @@ void qsieve_factor(fmpz_factor_t factors, const fmpz_t n) _fmpz_vec_clear(facs, 100); - qs_inf->siqs = (FLINT_FILE *) fopen(qs_inf->fname, "w"); + qs_inf->siqs = (FLINT_FILE *) fopen(qs_inf->fname, "wb"); if (qs_inf->siqs == NULL) flint_throw(FLINT_ERROR, "fopen fail\n"); qs_inf->num_primes = num_primes; /* linear algebra adjusts this */ diff --git a/src/qsieve/large_prime_variant.c b/src/qsieve/large_prime_variant.c index 3b93d3ee3c..aea96a493f 100644 --- a/src/qsieve/large_prime_variant.c +++ b/src/qsieve/large_prime_variant.c @@ -567,7 +567,7 @@ int qsieve_process_relation(qs_t qs_inf) if (qs_inf->siqs != NULL && fclose((FILE *) qs_inf->siqs)) flint_throw(FLINT_ERROR, "fclose fail\n"); - qs_inf->siqs = (FLINT_FILE *) fopen(qs_inf->fname, "r"); + qs_inf->siqs = (FLINT_FILE *) fopen(qs_inf->fname, "rb"); if (qs_inf->siqs == NULL) flint_throw(FLINT_ERROR, "fopen fail\n"); @@ -665,7 +665,7 @@ int qsieve_process_relation(qs_t qs_inf) done = 0; if (qs_inf->siqs != NULL && fclose((FILE *) qs_inf->siqs)) flint_throw(FLINT_ERROR, "fclose fail\n"); - qs_inf->siqs = (FLINT_FILE *) fopen(qs_inf->fname, "a"); + qs_inf->siqs = (FLINT_FILE *) fopen(qs_inf->fname, "ab"); if (qs_inf->siqs == NULL) flint_throw(FLINT_ERROR, "fopen fail\n"); } else From 2ce0e0254dc1e9752ab0336f4ab77f593c2a9b23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albin=20Ahlb=C3=A4ck?= Date: Wed, 15 Nov 2023 14:18:34 +0100 Subject: [PATCH 3/4] Change printf to flint_printf in qsieve --- src/qsieve.h | 2 +- src/qsieve/collect_relations.c | 6 +++--- src/qsieve/compute_poly_data.c | 16 ++++++++-------- src/qsieve/factor.c | 4 ++-- src/qsieve/large_prime_variant.c | 8 ++++---- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/qsieve.h b/src/qsieve.h index ab4377e1bd..5e4b76af12 100644 --- a/src/qsieve.h +++ b/src/qsieve.h @@ -18,7 +18,7 @@ #include "fmpz_types.h" #ifdef __cplusplus - extern "C" { +extern "C" { #endif #define QS_DEBUG 0 /* level of debug information printed, 0 = none */ diff --git a/src/qsieve/collect_relations.c b/src/qsieve/collect_relations.c index 421a59770b..9b9bed1521 100644 --- a/src/qsieve/collect_relations.c +++ b/src/qsieve/collect_relations.c @@ -242,9 +242,9 @@ slong qsieve_evaluate_candidate(qs_t qs_inf, ulong i, unsigned char * sieve, qs_ fmpz_add(res, res, C); /* res = AX^2 + 2BX + C */ #if QS_DEBUG & 128 - printf("res = "); fmpz_print(res); printf("\n"); + flint_printf("res = "); fmpz_print(res); flint_printf("\n"); flint_printf("Poly: "); fmpz_print(qs_inf->A); flint_printf("*x^2 + 2*"); - fmpz_print(poly->B); flint_printf("*x + "); fmpz_print(C); printf("\n"); + fmpz_print(poly->B); flint_printf("*x + "); fmpz_print(C); flint_printf("\n"); flint_printf("x = %wd\n", i - qs_inf->sieve_size / 2); #endif @@ -356,7 +356,7 @@ slong qsieve_evaluate_candidate(qs_t qs_inf, ulong i, unsigned char * sieve, qs_ { #if QS_DEBUG if (qs_inf->full_relation % 100 == 0) - printf("%ld relations\n", qs_inf->full_relation); + flint_printf("%ld relations\n", qs_inf->full_relation); #endif /* set sign amongst small factors */ if (fmpz_cmp_si(res, -1) == 0) diff --git a/src/qsieve/compute_poly_data.c b/src/qsieve/compute_poly_data.c index c77fa529cd..74f33d30e1 100644 --- a/src/qsieve/compute_poly_data.c +++ b/src/qsieve/compute_poly_data.c @@ -285,11 +285,11 @@ int qsieve_init_A(qs_t qs_inf) first_subset[j] = curr_subset[j]; /* save 1st tuple for restarts */ #if QS_DEBUG - printf("First A_ind = ("); + flint_printf("First A_ind = ("); for (i = 0; i < s - 1; i++) - printf("%ld, ", A_ind[i]); - printf("%ld", A_ind[s - 1]); - printf(")\n"); + flint_printf("%ld, ", A_ind[i]); + flint_printf("%ld", A_ind[s - 1]); + flint_printf(")\n"); #endif } @@ -509,11 +509,11 @@ int qsieve_next_A(qs_t qs_inf) } #if QS_DEBUG - printf("A_ind = ("); + flint_printf("A_ind = ("); for (i = 0; i < s - 1; i++) - printf("%ld, ", A_ind[i]); - printf("%ld", A_ind[s - 1]); - printf(")\n"); + flint_printf("%ld, ", A_ind[i]); + flint_printf("%ld", A_ind[s - 1]); + flint_printf(")\n"); #endif qs_inf->h = h; diff --git a/src/qsieve/factor.c b/src/qsieve/factor.c index dd75a71e67..c815a72862 100644 --- a/src/qsieve/factor.c +++ b/src/qsieve/factor.c @@ -418,7 +418,7 @@ void qsieve_factor(fmpz_factor_t factors, const fmpz_t n) more_primes: /* ran out of A's in init/sieving of linalg failed, increase FB */ #if QS_DEBUG - printf("Increasing factor base.\n"); + flint_printf("Increasing factor base.\n"); #endif delta = qs_inf->num_primes / 10; @@ -442,7 +442,7 @@ void qsieve_factor(fmpz_factor_t factors, const fmpz_t n) qs_inf->s = 0; /* indicate polynomials need setting up again */ #if QS_DEBUG - printf("Now %ld primes\n", qs_inf->num_primes); + flint_printf("Now %ld primes\n", qs_inf->num_primes); #endif if (small_factor) diff --git a/src/qsieve/large_prime_variant.c b/src/qsieve/large_prime_variant.c index aea96a493f..f5b655fccf 100644 --- a/src/qsieve/large_prime_variant.c +++ b/src/qsieve/large_prime_variant.c @@ -572,7 +572,7 @@ int qsieve_process_relation(qs_t qs_inf) flint_throw(FLINT_ERROR, "fopen fail\n"); #if QS_DEBUG & 64 - printf("Getting relations\n"); + flint_printf("Getting relations\n"); #endif while (1) @@ -613,14 +613,14 @@ int qsieve_process_relation(qs_t qs_inf) qs_inf->siqs = NULL; #if QS_DEBUG & 64 - printf("Removing duplicates\n"); + flint_printf("Removing duplicates\n"); #endif num_relations = qsieve_remove_duplicates(rel_list, num_relations); rel_list_length = num_relations; #if QS_DEBUG & 64 - printf("Merging relations\n"); + flint_printf("Merging relations\n"); #endif rlist = flint_malloc(num_relations * sizeof(relation_t)); @@ -656,7 +656,7 @@ int qsieve_process_relation(qs_t qs_inf) num_relations = rlist_length; #if QS_DEBUG & 64 - printf("Sorting relations\n"); + flint_printf("Sorting relations\n"); #endif if (rlist_length < qs_inf->num_primes + qs_inf->ks_primes + qs_inf->extra_rels) From be04d866ee86b65a65acda72322bbb3e7be3994c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albin=20Ahlb=C3=A4ck?= Date: Wed, 15 Nov 2023 16:51:13 +0100 Subject: [PATCH 4/4] Use fmpz_set_ui in file handling of qsieve --- src/qsieve/large_prime_variant.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/qsieve/large_prime_variant.c b/src/qsieve/large_prime_variant.c index f5b655fccf..9c18c4f702 100644 --- a/src/qsieve/large_prime_variant.c +++ b/src/qsieve/large_prime_variant.c @@ -292,17 +292,9 @@ relation_t qsieve_parse_relation(qs_t qs_inf) #if COEFF_MAX != -COEFF_MIN # error #endif - if (abslimb <= (mp_limb_t) COEFF_MAX) - { - *rel.Y = (Ysz < 0) ? -abslimb : abslimb; - } - else - { - mpz_ptr mY = _fmpz_new_mpz(); - mY->_mp_size = Ysz; - *mY->_mp_d = abslimb; - *rel.Y = PTR_TO_COEFF(mY); - } + fmpz_set_ui(rel.Y, abslimb); + if (Ysz < 0) + fmpz_neg(rel.Y, rel.Y); } else {