From c7324f8ae5b36d8defafa3acda42ffc663241042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= Date: Fri, 10 Jan 2025 10:49:45 +0100 Subject: [PATCH] feat: new import_images.pl script --- cgi/product_image_move.pl | 2 +- cgi/product_image_upload.pl | 2 +- cgi/product_multilingual.pl | 2 +- cgi/search.pl | 4 +- lib/ProductOpener/Import.pm | 538 +++++++++++++++++-------------- lib/ProductOpener/Products.pm | 22 +- scripts/check_photos.pl | 4 +- scripts/import_images.pl | 258 +++++++++++++++ scripts/import_systemeu.pl | 5 +- scripts/tag_stores_magasins_u.pl | 4 +- 10 files changed, 569 insertions(+), 272 deletions(-) create mode 100755 scripts/import_images.pl diff --git a/cgi/product_image_move.pl b/cgi/product_image_move.pl index 874d2a220495a..5701cd6792c7c 100644 --- a/cgi/product_image_move.pl +++ b/cgi/product_image_move.pl @@ -145,7 +145,7 @@ exit(0); } -my $product_ref = product_exists($product_id); # returns 0 if not +my $product_ref = retrieve_product($product_id); if (not $product_ref) { $log->warn("product does not exist", {code => $code, product_id => $product_id}); diff --git a/cgi/product_image_upload.pl b/cgi/product_image_upload.pl index c72fd42ae92ee..f768440a40b9b 100755 --- a/cgi/product_image_upload.pl +++ b/cgi/product_image_upload.pl @@ -197,7 +197,7 @@ exit(0); } - my $product_ref = product_exists($product_id); # returns 0 if not + my $product_ref = retrieve_product($product_id); if (not $product_ref) { $log->info("product code does not exist yet, creating product", {code => $code}); diff --git a/cgi/product_multilingual.pl b/cgi/product_multilingual.pl index d6d2c415c14da..cff15d02f51f4 100755 --- a/cgi/product_multilingual.pl +++ b/cgi/product_multilingual.pl @@ -251,7 +251,7 @@ ($product_ref) $product_id = product_id_for_owner($Owner_id, $code); $log->debug("we have a code", {code => $code, product_id => $product_id}) if $log->is_debug(); - $product_ref = product_exists($product_id); # returns 0 if not + $product_ref = retrieve_product($product_id); if ($product_ref) { $log->info("product exists, redirecting to page", {code => $code}) if $log->is_info(); diff --git a/cgi/search.pl b/cgi/search.pl index 0e78ce4f6de98..b8fc4059358a0 100755 --- a/cgi/search.pl +++ b/cgi/search.pl @@ -32,7 +32,7 @@ use ProductOpener::Display qw/:all/; use ProductOpener::HTTP qw/write_cors_headers/; use ProductOpener::Users qw/$Owner_id/; -use ProductOpener::Products qw/normalize_code normalize_search_terms product_exists product_id_for_owner product_url/; +use ProductOpener::Products qw/normalize_code normalize_search_terms retrieve_product product_id_for_owner product_url/; use ProductOpener::Food qw/%nutriments_lists/; use ProductOpener::Tags qw/:all/; use ProductOpener::PackagerCodes qw/normalize_packager_codes/; @@ -166,7 +166,7 @@ if ((defined $code) and (length($code) > 0)) { my $product_id = product_id_for_owner($Owner_id, $code); - my $product_ref = product_exists($product_id); # returns 0 if not + my $product_ref = retrieve_product($product_id); if ($product_ref) { $log->info("product code exists, redirecting to product page", {code => $code}); diff --git a/lib/ProductOpener/Import.pm b/lib/ProductOpener/Import.pm index 2661351570b82..7910b4475a803 100644 --- a/lib/ProductOpener/Import.pm +++ b/lib/ProductOpener/Import.pm @@ -71,6 +71,8 @@ BEGIN { &import_csv_file &import_products_categories_from_public_database + &list_product_images_files_in_dir + &upload_images_for_product ); # symbols to export on request %EXPORT_TAGS = (all => [@EXPORT_OK]); @@ -126,7 +128,27 @@ $IMPORT_MAX_PACKAGING_COMPONENTS = 10; # image_dir: path to image directory # stats: stats map # return -sub import_images_from_dir ($image_dir, $stats) { + +=head2 list_product_images_files_in_dir ($image_dir, $stats) + +List product images files in a directory. +The resulting list can be used by import_images_for_product() to upload the images. + +=head3 Arguments + +=head4 $image_dir Image directory + +=head4 $stats Stats reference + +=head3 Returns + +A reference to a hash of product codes with image types (front, ingredients, nutrition, other) with optional language codes as keys, +and image file names as values. + +=cut + +sub list_product_images_files_in_dir ($image_dir, $stats) { + my $images_ref = {}; if (not -d $image_dir) { @@ -239,6 +261,273 @@ sub import_images_from_dir ($image_dir, $stats) { return $images_ref; } +=head2 upload_images_for_product ($args_ref, $images_ref, $product_ref, $imported_product_ref, $product_id, $code, $user_id, $comment, $stats_ref) + +Given a list of images for a product with possible image types (front, ingredients, nutrition, other), +upload them to the server and select the image type. + +This function is called by import_csv_file() as CSV file can contain links to images, or we may have associated images in a directory +(loaded with list_product_images_files_in_dir()). + +It is is also called directly by the import_images.pl script, to upload images from a directory. + +=head3 Arguments + +=head4 $args_ref Import arguments reference + +=head4 $images_ref Images reference for the product + +=head4 $product_ref Product reference + +Only needed to get the "images" field, and "lc" field to determine the language of the product. + +=head4 $imported_product_ref Imported product reference: from a CSV file, may contain image coordinates + +=head4 $product_id Product ID + +=head4 $code Product code + +=head4 $user_id User ID used to upload and select the pictures + +=head4 $stats_ref Stats reference to keep track of the number of images added + +=cut + +sub upload_images_for_product($args_ref, $images_ref, $product_ref, $imported_product_ref, $product_id, $code, + $user_id, $comment, $stats_ref) +{ + + if ( (not $args_ref->{test}) + and (not((defined $args_ref->{do_not_upload_images}) and ($args_ref->{do_not_upload_images})))) + { + + $log->debug("uploading images for product", + {code => $code, images_ref => $images_ref, product_ref => $product_ref}) + if $log->is_debug(); + + # Keep track of the images we select so that we don't select multiple images for the same field + my %selected_images = (); + + foreach my $imagefield (sort keys %{$images_ref}) { + + $log->debug("uploading image for product", {imagefield => $imagefield, code => $code}) + if $log->is_debug(); + + my $current_max_imgid = -1; + + if (defined $product_ref->{images}) { + foreach my $imgid (keys %{$product_ref->{images}}) { + if (($imgid =~ /^\d/) and ($imgid > $current_max_imgid)) { + $current_max_imgid = $imgid; + } + } + } + + # if the language is not specified, assign it to the language of the product + + my $imagefield_with_lc = $imagefield; + + # image_other_url.2 -> remove the number + $imagefield_with_lc =~ s/(\.|_)(\d+)$//; + + if ($imagefield_with_lc !~ /_\w\w/) { + $imagefield_with_lc .= "_" . $product_ref->{lc}; + } + + # upload the image + my $file = $images_ref->{$imagefield}; + + # Skip PDF file has we have issues to convert them, and they are sometimes not images about the product + # but multi-pages product sheets, certificates etc. + if ($file =~ /\.pdf$/) { + $log->debug("skipping PDF file", {file => $file, imagefield => $imagefield, code => $code}) + if $log->is_debug(); + } + elsif (-e "$file") { + $log->debug("found image file", {file => $file, imagefield => $imagefield, code => $code}) + if $log->is_debug(); + + # upload a photo + my $imgid; + my $debug; + my $return_code + = process_image_upload($product_id, "$file", $user_id, undef, $comment, \$imgid, \$debug); + $log->debug( + "process_image_upload", + { + file => $file, + imagefield => $imagefield, + code => $code, + return_code => $return_code, + imgid => $imgid, + imagefield_with_lc => $imagefield_with_lc, + debug => $debug + } + ) if $log->is_debug(); + + if (($imgid > 0) and ($imgid > $current_max_imgid)) { + $stats_ref->{products_images_added}{$code} = 1; + } + + my $x1 = $imported_product_ref->{"image_" . $imagefield . "_x1"} || -1; + my $y1 = $imported_product_ref->{"image_" . $imagefield . "_y1"} || -1; + my $x2 = $imported_product_ref->{"image_" . $imagefield . "_x2"} || -1; + my $y2 = $imported_product_ref->{"image_" . $imagefield . "_y2"} || -1; + my $coordinates_image_size + = $imported_product_ref->{"image_" . $imagefield . "_coordinates_image_size"} || $crop_size; + my $angle = $imported_product_ref->{"image_" . $imagefield . "_angle"} || 0; + my $normalize = $imported_product_ref->{"image_" . $imagefield . "_normalize"} || "false"; + my $white_magic = $imported_product_ref->{"image_" . $imagefield . "_white_magic"} || "false"; + + $log->debug( + "select and crop image?", + { + code => $code, + imgid => $imgid, + current_max_imgid => $current_max_imgid, + imagefield_with_lc => $imagefield_with_lc, + x1 => $x1, + y1 => $y1, + x2 => $x2, + y2 => $y2, + angle => $angle, + normalize => $normalize, + white_magic => $white_magic + } + ) if $log->is_debug(); + + # select the photo + if ( + ($imagefield_with_lc =~ /front|ingredients|nutrition|packaging/) + and ( + ( + not( (defined $args_ref->{only_select_not_existing_images}) + and ($args_ref->{only_select_not_existing_images})) + ) + or ( (not defined $product_ref->{images}) + or (not defined $product_ref->{images}{$imagefield_with_lc})) + ) + ) + { + + if (($imgid > 0) and ($imgid > $current_max_imgid)) { + + $log->debug( + "assigning image imgid to imagefield_with_lc", + { + code => $code, + current_max_imgid => $current_max_imgid, + imgid => $imgid, + imagefield_with_lc => $imagefield_with_lc, + x1 => $x1, + y1 => $y1, + x2 => $x2, + y2 => $y2, + angle => $angle, + normalize => $normalize, + white_magic => $white_magic + } + ) if $log->is_debug(); + $selected_images{$imagefield_with_lc} = 1; + eval { + process_image_crop($user_id, $product_id, $imagefield_with_lc, $imgid, $angle, + $normalize, $white_magic, $x1, $y1, $x2, $y2, $coordinates_image_size); + }; + } + else { + $log->debug("returned imgid $imgid not greater than the previous max imgid: $current_max_imgid", + {imgid => $imgid, current_max_imgid => $current_max_imgid}) + if $log->is_debug(); + + # overwrite already selected images + # if the selected image is not the same + # or if we have non null crop coordinates that differ + if ( + ($imgid > 0) + and (exists $product_ref->{images}) + and ( + (not exists $product_ref->{images}{$imagefield_with_lc}) + or ( + ( + ($product_ref->{images}{$imagefield_with_lc}{imgid} != $imgid) + or ( ($x1 > 1) + and ($product_ref->{images}{$imagefield_with_lc}{x1} != $x1)) + or ( ($x2 > 1) + and ($product_ref->{images}{$imagefield_with_lc}{x2} != $x2)) + or ( ($y1 > 1) + and ($product_ref->{images}{$imagefield_with_lc}{y1} != $y1)) + or ( ($y2 > 1) + and ($product_ref->{images}{$imagefield_with_lc}{y2} != $y2)) + or ($product_ref->{images}{$imagefield_with_lc}{angle} != $angle) + ) + ) + ) + ) + { + $log->debug( + "re-assigning image imgid to imagefield_with_lc", + { + code => $code, + imgid => $imgid, + imagefield_with_lc => $imagefield_with_lc, + x1 => $x1, + y1 => $y1, + x2 => $x2, + y2 => $y2, + coordinates_image_size => $coordinates_image_size, + angle => $angle, + normalize => $normalize, + white_magic => $white_magic + } + ) if $log->is_debug(); + $selected_images{$imagefield_with_lc} = 1; + eval { + process_image_crop($user_id, $product_id, $imagefield_with_lc, $imgid, $angle, + $normalize, $white_magic, $x1, $y1, $x2, $y2, $coordinates_image_size); + }; + } + } + } + # If the image type is "other" and we don't have a front image, assign it + # This is in particular for producers that send us many images without specifying their type: assume the first one is the front + elsif ( ($imgid > 0) + and ($imagefield_with_lc =~ /^other/) + and (not defined $product_ref->{images}{"front_" . $product_ref->{lc}}) + and (not defined $selected_images{"front_" . $product_ref->{lc}})) + { + $log->debug( + "selecting front image as we don't have one", + { + imgid => $imgid, + imagefield => $imagefield, + front_imagefield => "front_" . $product_ref->{lc}, + x1 => $x1, + y1 => $y1, + x2 => $x2, + y2 => $y2, + coordinates_image_size => $coordinates_image_size, + angle => $angle, + normalize => $normalize, + white_magic => $white_magic + } + ) if $log->is_debug(); + # Keep track that we have selected an image, so that we don't select another one after, + # as we don't reload the product_ref after calling process_image_crop() + $selected_images{"front_" . $product_ref->{lc}} = 1; + eval { + process_image_crop($user_id, $product_id, "front_" . $product_ref->{lc}, + $imgid, $angle, $normalize, $white_magic, $x1, $y1, $x2, $y2, $coordinates_image_size); + }; + } + } + else { + $log->debug("did not find image file", {file => $file, imagefield => $imagefield, code => $code}) + if $log->is_debug(); + } + } + } +} + # download image at given url parameter sub download_image ($image_url) { @@ -1433,7 +1722,7 @@ sub import_csv_file ($args_ref) { # Read images from directory if supplied my $images_ref = {}; if ((defined $args_ref->{images_dir}) and ($args_ref->{images_dir} ne '')) { - $images_ref = import_images_from_dir($args_ref->{images_dir}, $stats_ref); + $images_ref = list_product_images_files_in_dir($args_ref->{images_dir}, $stats_ref); } $log->debug("importing products", {csv_file => $args_ref->{csv_file}}) if $log->is_debug(); @@ -1877,7 +2166,7 @@ sub import_csv_file ($args_ref) { } # TODO: check what happens if the product exists with a different product type than the current server - my $product_ref = product_exists($product_id); # returns 0 if not + my $product_ref = retrieve_product($product_id); my $product_comment = $args_ref->{comment}; if ((defined $imported_product_ref->{comment}) and ($imported_product_ref->{comment} ne "")) { @@ -2496,246 +2785,11 @@ sub import_csv_file ($args_ref) { # Upload images if (defined $images_ref->{$code}) { - $stats_ref->{products_with_images}{$code} = 1; - - if ( (not $args_ref->{test}) - and (not((defined $args_ref->{do_not_upload_images}) and ($args_ref->{do_not_upload_images})))) - { - - $log->debug("uploading images for product", {code => $code}) if $log->is_debug(); - - my $images_ref = $images_ref->{$code}; - - # Keep track of the images we select so that we don't select multiple images for the same field - my %selected_images = (); - - foreach my $imagefield (sort keys %{$images_ref}) { - - $log->debug("uploading image for product", {imagefield => $imagefield, code => $code}) - if $log->is_debug(); - - my $current_max_imgid = -1; - - if (defined $product_ref->{images}) { - foreach my $imgid (keys %{$product_ref->{images}}) { - if (($imgid =~ /^\d/) and ($imgid > $current_max_imgid)) { - $current_max_imgid = $imgid; - } - } - } - - # if the language is not specified, assign it to the language of the product - - my $imagefield_with_lc = $imagefield; - - # image_other_url.2 -> remove the number - $imagefield_with_lc =~ s/(\.|_)(\d+)$//; - - if ($imagefield_with_lc !~ /_\w\w/) { - $imagefield_with_lc .= "_" . $product_ref->{lc}; - } - - # upload the image - my $file = $images_ref->{$imagefield}; - - # Skip PDF file has we have issues to convert them, and they are sometimes not images about the product - # but multi-pages product sheets, certificates etc. - if ($file =~ /\.pdf$/) { - $log->debug("skipping PDF file", {file => $file, imagefield => $imagefield, code => $code}) - if $log->is_debug(); - } - elsif (-e "$file") { - $log->debug("found image file", {file => $file, imagefield => $imagefield, code => $code}) - if $log->is_debug(); - - # upload a photo - my $imgid; - my $debug; - my $return_code - = process_image_upload($product_id, "$file", $user_id, undef, $product_comment, \$imgid, - \$debug); - $log->debug( - "process_image_upload", - { - file => $file, - imagefield => $imagefield, - code => $code, - return_code => $return_code, - imgid => $imgid, - imagefield_with_lc => $imagefield_with_lc, - debug => $debug - } - ) if $log->is_debug(); - - if (($imgid > 0) and ($imgid > $current_max_imgid)) { - $stats_ref->{products_images_added}{$code} = 1; - } - - my $x1 = $imported_product_ref->{"image_" . $imagefield . "_x1"} || -1; - my $y1 = $imported_product_ref->{"image_" . $imagefield . "_y1"} || -1; - my $x2 = $imported_product_ref->{"image_" . $imagefield . "_x2"} || -1; - my $y2 = $imported_product_ref->{"image_" . $imagefield . "_y2"} || -1; - my $coordinates_image_size - = $imported_product_ref->{"image_" . $imagefield . "_coordinates_image_size"} || $crop_size; - my $angle = $imported_product_ref->{"image_" . $imagefield . "_angle"} || 0; - my $normalize = $imported_product_ref->{"image_" . $imagefield . "_normalize"} || "false"; - my $white_magic = $imported_product_ref->{"image_" . $imagefield . "_white_magic"} || "false"; - - $log->debug( - "select and crop image?", - { - code => $code, - imgid => $imgid, - current_max_imgid => $current_max_imgid, - imagefield_with_lc => $imagefield_with_lc, - x1 => $x1, - y1 => $y1, - x2 => $x2, - y2 => $y2, - angle => $angle, - normalize => $normalize, - white_magic => $white_magic - } - ) if $log->is_debug(); - - # select the photo - if ( - ($imagefield_with_lc =~ /front|ingredients|nutrition|packaging/) - and ( - ( - not( (defined $args_ref->{only_select_not_existing_images}) - and ($args_ref->{only_select_not_existing_images})) - ) - or ( (not defined $product_ref->{images}) - or (not defined $product_ref->{images}{$imagefield_with_lc})) - ) - ) - { - - if (($imgid > 0) and ($imgid > $current_max_imgid)) { - - $log->debug( - "assigning image imgid to imagefield_with_lc", - { - code => $code, - current_max_imgid => $current_max_imgid, - imgid => $imgid, - imagefield_with_lc => $imagefield_with_lc, - x1 => $x1, - y1 => $y1, - x2 => $x2, - y2 => $y2, - angle => $angle, - normalize => $normalize, - white_magic => $white_magic - } - ) if $log->is_debug(); - $selected_images{$imagefield_with_lc} = 1; - eval { - process_image_crop($user_id, $product_id, $imagefield_with_lc, $imgid, $angle, - $normalize, $white_magic, $x1, $y1, $x2, $y2, $coordinates_image_size); - }; - # $modified++; - - } - else { - $log->debug( - "returned imgid $imgid not greater than the previous max imgid: $current_max_imgid", - {imgid => $imgid, current_max_imgid => $current_max_imgid} - ) if $log->is_debug(); - - # overwrite already selected images - # if the selected image is not the same - # or if we have non null crop coordinates that differ - if ( - ($imgid > 0) - and (exists $product_ref->{images}) - and ( - (not exists $product_ref->{images}{$imagefield_with_lc}) - or ( - ( - ($product_ref->{images}{$imagefield_with_lc}{imgid} != $imgid) - or ( ($x1 > 1) - and ($product_ref->{images}{$imagefield_with_lc}{x1} != $x1)) - or ( ($x2 > 1) - and ($product_ref->{images}{$imagefield_with_lc}{x2} != $x2)) - or ( ($y1 > 1) - and ($product_ref->{images}{$imagefield_with_lc}{y1} != $y1)) - or ( ($y2 > 1) - and ($product_ref->{images}{$imagefield_with_lc}{y2} != $y2)) - or ($product_ref->{images}{$imagefield_with_lc}{angle} != $angle) - ) - ) - ) - ) - { - $log->debug( - "re-assigning image imgid to imagefield_with_lc", - { - code => $code, - imgid => $imgid, - imagefield_with_lc => $imagefield_with_lc, - x1 => $x1, - y1 => $y1, - x2 => $x2, - y2 => $y2, - coordinates_image_size => $coordinates_image_size, - angle => $angle, - normalize => $normalize, - white_magic => $white_magic - } - ) if $log->is_debug(); - $selected_images{$imagefield_with_lc} = 1; - eval { - process_image_crop($user_id, $product_id, $imagefield_with_lc, $imgid, $angle, - $normalize, $white_magic, $x1, $y1, $x2, $y2, $coordinates_image_size); - }; - # $modified++; - } - - } - } - # If the image type is "other" and we don't have a front image, assign it - # This is in particular for producers that send us many images without specifying their type: assume the first one is the front - elsif ( ($imgid > 0) - and ($imagefield_with_lc =~ /^other/) - and (not defined $product_ref->{images}{"front_" . $product_ref->{lc}}) - and (not defined $selected_images{"front_" . $product_ref->{lc}})) - { - $log->debug( - "selecting front image as we don't have one", - { - imgid => $imgid, - imagefield => $imagefield, - front_imagefield => "front_" . $product_ref->{lc}, - x1 => $x1, - y1 => $y1, - x2 => $x2, - y2 => $y2, - coordinates_image_size => $coordinates_image_size, - angle => $angle, - normalize => $normalize, - white_magic => $white_magic - } - ) if $log->is_debug(); - # Keep track that we have selected an image, so that we don't select another one after, - # as we don't reload the product_ref after calling process_image_crop() - $selected_images{"front_" . $product_ref->{lc}} = 1; - eval { - process_image_crop($user_id, $product_id, "front_" . $product_ref->{lc}, - $imgid, $angle, $normalize, $white_magic, $x1, $y1, $x2, $y2, - $coordinates_image_size); - }; - } - } - else { - $log->debug("did not find image file", - {file => $file, imagefield => $imagefield, code => $code}) - if $log->is_debug(); - } - } - } + upload_images_for_product( + $args_ref, $images_ref->{$code}, $product_ref, $imported_product_ref, $product_id, + $code, $user_id, $product_comment, $stats_ref + ); } else { $log->debug("no images for product", {code => $code}) if $log->is_debug(); diff --git a/lib/ProductOpener/Products.pm b/lib/ProductOpener/Products.pm index 95ba9f34c6be9..5044332652e5f 100644 --- a/lib/ProductOpener/Products.pm +++ b/lib/ProductOpener/Products.pm @@ -76,7 +76,6 @@ BEGIN { &product_path &product_path_from_id &product_id_from_path - &product_exists &get_owner_id &normalize_product_data &init_product @@ -716,20 +715,6 @@ sub product_id_from_path ($product_path) { return $id; } -sub product_exists ($product_id) { - - # deprecated, just use retrieve_product() - - my $product_ref = retrieve_product($product_id); - - if (not defined $product_ref) { - return 0; - } - else { - return $product_ref; - } -} - sub get_owner_id ($userid, $orgid, $ownerid) { if ($server_options{private_products}) { @@ -2909,12 +2894,13 @@ sub compute_codes ($product_ref) { my $ean = undef; + # Note: we now normalize codes, so we should not have conflicts if (length($code) == 12) { $ean = '0' . $code; - if (product_exists('0' . $code)) { + if (retrieve_product('0' . $code)) { push @codes, "conflict-with-ean-13"; } - elsif (-e ("$BASE_DIRS{PRODUCTS}/" . product_path_from_id("0" . $code))) { + elsif (retrieve_product('0' . $code), 1) { push @codes, "conflict-with-deleted-ean-13"; } } @@ -2923,7 +2909,7 @@ sub compute_codes ($product_ref) { $ean = $code; my $upc = $code; $upc =~ s/^.//; - if (product_exists($upc)) { + if (retrieve_product($upc)) { push @codes, "conflict-with-upc-12"; } } diff --git a/scripts/check_photos.pl b/scripts/check_photos.pl index 88c3d61fc10b1..f66e4de2b5d61 100755 --- a/scripts/check_photos.pl +++ b/scripts/check_photos.pl @@ -34,7 +34,7 @@ use ProductOpener::Images qw/scan_code/; use ProductOpener::Lang qw/:all/; use ProductOpener::Mail qw/:all/; -use ProductOpener::Products qw/product_exists/; +use ProductOpener::Products qw/retrieve_product/; use ProductOpener::Food qw/:all/; use ProductOpener::Ingredients qw/:all/; use ProductOpener::Images qw/:all/; @@ -88,7 +88,7 @@ $j++; - my $product_ref = product_exists($code); + my $product_ref = retrieve_product($code); if ($product_ref) { $exists++; print "code $code exists\n"; diff --git a/scripts/import_images.pl b/scripts/import_images.pl new file mode 100755 index 0000000000000..ef186833cfb40 --- /dev/null +++ b/scripts/import_images.pl @@ -0,0 +1,258 @@ +#!/usr/bin/perl -w + +# This file is part of Product Opener. +# +# Product Opener +# Copyright (C) 2011-2023 Association Open Food Facts +# Contact: contact@openfoodfacts.org +# Address: 21 rue des Iles, 94100 Saint-Maur des Fossés, France +# +# Product Opener is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +use ProductOpener::PerlStandards; + +use Log::Any qw($log); + +use ProductOpener::Config qw/:all/; +use ProductOpener::Paths qw/%BASE_DIRS/; +use ProductOpener::Store qw/:all/; +use ProductOpener::Index qw/:all/; +use ProductOpener::Display qw/:all/; +use ProductOpener::Tags qw/:all/; +use ProductOpener::Images qw/:all/; +use ProductOpener::Lang qw/:all/; +use ProductOpener::Mail qw/send_email_to_producers_admin/; +use ProductOpener::Products qw/:all/; +use ProductOpener::Users qw/:all/; +use ProductOpener::Import qw/list_product_images_files_in_dir upload_images_for_product/; +use ProductOpener::LoadData qw/load_data/; + +use Storable qw/dclone/; +use Encode; +use Time::Local; +use Data::Dumper; +use Getopt::Long; + +use Log::Any::Adapter 'TAP', filter => "none"; + +my $usage = < \$import_lc, + "csv_file=s" => \$csv_file, + "images_dir=s" => \$images_dir, + "user_id=s" => \$user_id, + "org_id=s" => \$org_id, + "owner_id=s" => \$owner_id, + "comment=s" => \$comment, + "source_id=s" => \$source_id, + "source_name=s" => \$source_name, + "source_url=s" => \$source_url, + "source_licence=s" => \$source_licence, + "source_licence_url=s" => \$source_licence_url, + "test" => \$test, + "manufacturer" => \$manufacturer, + "no_source" => \$no_source, +) or die("Error in command line arguments:\n\n$usage"); + +print STDERR "import_csv_file.pl +- import_lc: $import_lc +- user_id: $user_id +- org_id: $org_id +- owner_id: $owner_id +- images_dir: $images_dir +- comment: $comment +- source_id: $source_id +- source_name: $source_name +- source_url: $source_url +- source_licence: $source_licence +- source_licence_url: $source_licence_url +- manufacturer: $manufacturer +- testing: $test +"; + +my $missing_arg = 0; + +if (not defined $import_lc) { + print STDERR "missing --import_lc parameter\n"; + $missing_arg++; +} + +if (not defined $images_dir) { + print STDERR "missing --images_dir parameter\n"; + $missing_arg++; +} + +if (not defined $user_id) { + print STDERR "missing --user_id parameter\n"; + $missing_arg++; +} + +if (not $no_source) { + + if (not defined $source_id) { + print STDERR "missing --source_id parameter\n"; + $missing_arg++; + } + + if (not defined $source_name) { + print STDERR "missing --source_name parameter\n"; + $missing_arg++; + } + + if (not defined $source_url) { + print STDERR "missing --source_url parameter\n"; + $missing_arg++; + } +} + +$missing_arg and exit(); + +load_data(); + +my $args_ref = { + user_id => $user_id, + org_id => $org_id, + owner_id => $owner_id, + global_values => {lc => $import_lc}, + images_dir => $images_dir, + comment => $comment, + source_id => $source_id, + source_name => $source_name, + source_url => $source_url, + source_licence => $source_licence, + source_licence_url => $source_licence_url, + test => $test, + manufacturer => $manufacturer, + no_source => $no_source, +}; + +$Org_id = $org_id; +$Owner_id = get_owner_id($User_id, $Org_id, $args_ref->{owner_id}); + +my $stats_ref = { + products_created => {}, + products_with_images => {}, +}; + +my $images_ref = list_product_images_files_in_dir($args_ref->{images_dir}, $stats_ref); + +foreach my $code (sort keys %{$images_ref}) { + + print STDERR "code: $code\n"; + + my $product_id = product_id_for_owner($Owner_id, $code); + + my $product_ref = retrieve_product($product_id); + + if (not defined $product_ref) { + $stats_ref->{products_created}{$code} = 1; + + $product_ref = init_product($user_id, $org_id, $code, undef); + $product_ref->{lc} = $import_lc; + $product_ref->{lang} = $import_lc; + + delete $product_ref->{countries}; + delete $product_ref->{countries_tags}; + delete $product_ref->{countries_hierarchy}; + + store_product($user_id, $product_ref, "Creating product (import_images) - " . ($comment || "")); + } + + # Upload images + + $stats_ref->{products_with_images}{$code} = 1; + upload_images_for_product($args_ref, $images_ref->{$code}, $product_ref, {}, $product_id, $code, $user_id, + "Adding photo (import_images) - " . ($comment || ""), $stats_ref); +} + +print STDERR "\n\nstats:\n\n"; + +foreach my $stat (sort keys %{$stats_ref}) { + + print STDERR $stat . "\t" . (scalar keys %{$stats_ref->{$stat}}) . "\n"; + + open(my $out, ">", "$BASE_DIRS{CACHE_TMP}/import_images.$stat.txt") + or print "Could not create import_images.$stat.txt : $!\n"; + + foreach my $code (sort keys %{$stats_ref->{$stat}}) { + print $out $code . "\n"; + } + close($out); +} + +# Send an e-mail notification to admins + +my $template_data_ref = { + args => $args_ref, + stats => $stats_ref, +}; + +my $mail = ''; +process_template("emails/import_csv_file_admin_notification.tt.html", $template_data_ref, \$mail) + or print STDERR "emails/import_csv_file_admin_notification.tt.html template error: " . $tt->error(); +if ($mail =~ /^\s*Subject:\s*(.*)\n/i) { + my $subject = $1; + my $body = $'; + $body =~ s/^\n+//; + + send_email_to_producers_admin($subject, $body); + + print "email subject: $subject\n\n"; + print "email body:\n$body\n\n"; +} + +if ($stats_ref->{error}) { + print STDERR "An error occured: " . $stats_ref->{error}{error} . "\n"; + exit(1); +} diff --git a/scripts/import_systemeu.pl b/scripts/import_systemeu.pl index 810254f9531ad..89aebb6b9c626 100755 --- a/scripts/import_systemeu.pl +++ b/scripts/import_systemeu.pl @@ -36,8 +36,7 @@ use ProductOpener::Images qw/process_image_crop process_image_upload/; use ProductOpener::Lang qw/$lc lang/; use ProductOpener::Mail qw/:all/; -use ProductOpener::Products - qw/analyze_and_enrich_product_data init_product product_exists retrieve_product store_product/; +use ProductOpener::Products qw/analyze_and_enrich_product_data init_product retrieve_product store_product/; use ProductOpener::Food qw/:all/; use ProductOpener::Units qw/unit_to_g/; use ProductOpener::Ingredients qw/:all/; @@ -459,7 +458,7 @@ print "product $i - code: $code\n"; - my $product_ref = product_exists("org-systeme-u/" . $code); # returns 0 if not + my $product_ref = retrieve_product("org-systeme-u/" . $code); # returns 0 if not if (not $product_ref) { print "- does not exist in OFF yet\n"; diff --git a/scripts/tag_stores_magasins_u.pl b/scripts/tag_stores_magasins_u.pl index 965993292cb96..cc85a1604f9c9 100755 --- a/scripts/tag_stores_magasins_u.pl +++ b/scripts/tag_stores_magasins_u.pl @@ -36,7 +36,7 @@ use ProductOpener::Images qw/:all/; use ProductOpener::Lang qw/$lc/; use ProductOpener::Mail qw/:all/; -use ProductOpener::Products qw/product_exists store_product/; +use ProductOpener::Products qw/retrieve_product store_product/; use ProductOpener::Food qw/:all/; use ProductOpener::Ingredients qw/:all/; use ProductOpener::Images qw/:all/; @@ -140,7 +140,7 @@ print "PRODUCT LINE NUMBER $i - CODE $code"; - my $product_ref = product_exists($code); # returns 0 if not + my $product_ref = retrieve_product($code); # returns 0 if not if (not $product_ref) { print "- does not exist in OFF yet\n";