refactor: configuration for OBF with new code + ProductsFeatures.pm f…

…or features by product type (#10346)

* remove misplaced translations

* conf for obf

* add openbeautyfacts.com

* list of tags fields

* new ProductsFeatures.pm module to activate features by product type

* update tests

* temporarily put ingredients after nutrition, to ease tests comparison

* lint

* update tests

cgi/product_multilingual.pl

@@ -54,6 +54,7 @@
 use ProductOpener::Events qw/send_event/;
 use ProductOpener::API qw/get_initialized_response/;
 use ProductOpener::APIProductWrite qw/skip_protected_field/;
+use ProductOpener::ProductsFeatures qw/feature_enabled/;
 use ProductOpener::Orgs qw/update_import_date/;
 
 use Apache2::RequestRec ();

conf/apache-2.4/obf-ports.conf

@@ -0,0 +1,7 @@
+# If you just change the port or add more ports here, you will likely also
+# have to change the VirtualHost statement in
+# /etc/apache2/sites-enabled/000-default.conf
+
+Listen 8002
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

conf/apache-2.4/opf-ports.conf

@@ -0,0 +1,7 @@
+# If you just change the port or add more ports here, you will likely also
+# have to change the VirtualHost statement in
+# /etc/apache2/sites-enabled/000-default.conf
+
+Listen 8003
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

conf/apache-2.4/opff-ports.conf

@@ -0,0 +1,7 @@
+# If you just change the port or add more ports here, you will likely also
+# have to change the VirtualHost statement in
+# /etc/apache2/sites-enabled/000-default.conf
+
+Listen 8001
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

conf/nginx/sites-available/obf

@@ -1,140 +1,136 @@
-##
-# You should look at the following URL's in order to grasp a solid understanding
-# of Nginx configuration files in order to fully unleash the power of Nginx.
-# http://wiki.nginx.org/Pitfalls
-# http://wiki.nginx.org/QuickStart
-# http://wiki.nginx.org/Configuration
+# Default server configuration - nginx_status
 #
-# Generally, you will want to move this file somewhere, and start with a clean
-# file but keep this around for reference. Or just disable in sites-enabled.
-#
-# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples.
-##
-
+server {
+	listen 80 default_server;
+	listen [::]:80 default_server;
+	server_name _;
+
+	# nginx_status configuration, need for Munin
+	location /nginx_status {
+			stub_status on;
+			access_log   off;
+			allow all;
+	}
+}
 
-include /etc/nginx/snippets/expires-no-json-xml.conf;
+# variables definitions for expiry headers are loaded from /etc/nginx/conf.d/expires-no-json-xml.conf
 
-# Default server configuration
-#
 server {
-	# listen 80 default_server;
-	# listen [::]:80 default_server;
 
 	listen 80;
 	listen [::]:80;
 
-
-	server_name openbeautyfacts.org *.openbeautyfacts.org ;
-
-	# SSL configuration
-	#
-	# listen 443 ssl default_server;
-	# listen [::]:443 ssl default_server;
-	#
-	# Self signed certs generated by the ssl-cert package
-	# Don't use them in a production server!
-	#
-	# include snippets/snakeoil.conf;
-
-	listen 443 http2 ssl;
-	listen [::]:443 http2 ssl;
-
-	include snippets/ssl.openbeautyfacts.org;
-	include snippets/ssl-params.conf;
+	# Product Opener needs a root domain + a wildcard for all subdomains
+	# openbeautyfacts.com is also used for testing
+	server_name openbeautyfacts.org *.openbeautyfacts.org openbeautyfacts.com *.openbeautyfacts.com;
 
 	root /srv/obf/html;
 
-	access_log /srv/obf/logs/nginx.access2.log;
-	error_log /srv/obf/logs/nginx.error2.log;
+	# enable large uploads
+	client_max_body_size 20M;
+	client_body_timeout 120s;
+	client_header_timeout 120s;
 
-	# Redirect GET requests to https. POST requests will be transformed
-	# to GET by most browsers when redirected, and it breaks apps that
-	# use the API through http.
+	# logs location: default is static-obf, will be changed to proxy-obf
+	# for requests passed to Apache
+	access_log /var/log/nginx/static-obf-access.log proxied_requests buffer=256K flush=1s;
+	error_log /var/log/nginx/static-obf-error.log;
 
-	if ($scheme = http) {
-		set $test "A";
-	}
-	if ($request_uri !~ "/api/") {
-		set $test "${test}B";
-	}
-	if ($request_method = GET) {
-		set $test "${test}C";
-	}
-	if ($test = ABC) {
-		return 301 https://$host$request_uri;
-	}
+	# some redirection for specific subdomains
+	include snippets/obf.domain-redirects.include;
 
 	gzip on;
 	gzip_min_length 1000;
+	gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript text/csv;
 
-	location /data/ {
-		include snippets/off.cors-headers.include;
-		include snippets/expiry-headers.include;
-		include snippets/ssl-headers.conf;
-		# First attempt to serve request as file, then
-		# as directory, then fall back to displaying a 404.
-		try_files $uri $uri/ =404;
-	}
-
-	# Add index.php to the list if you are using PHP
 	index index.html index.htm index.nginx-debian.html;
 
-	location ~* \.(eot|ttf|woff|woff2)$ {
-		include snippets/off.cors-headers.include;
-		include snippets/expiry-headers.include;
-	}
-
 	location ~ ^/images/products/ {
 		include snippets/off.cors-headers.include;
+		include snippets/expiry-headers.include;
 		add_header Link "<http://creativecommons.org/licenses/by-sa/3.0/>; rel='license'; title='CC-BY-SA 3.0'";
 		# optimize gzip compressed content (like OCR .json stored next to .jpg files)
 		gzip_static always;
 		gunzip on;
 	}
 
+	if ($http_referer ~* (jobothoniel.com) ) { return 403; } # blocked since 2021-07-13
+
+	# the app requests /1.json to get the product count...
+	# the commented code below is to serve a static copy
+	# if there is a spike of installs
+	location ~ ^/1.json$ {
+			root /srv/obf/html/static;
+	try_files /1.json = 404;
+	}
+
+	# Static files are served directly by NGINX
 
 	location ~ ^/(favicon.ico) {
 		# First attempt to serve request as file, then
 		# as directory, then fall back to displaying a 404.
 		try_files $uri $uri/ =404;
 	}
 
-	location ~ ^/(.well-known|images|fonts|css|js|rss|files|resources|foundation|bower_components)/ {
+	# Static files are served directly by NGINX
+	location ~ ^/(.well-known|files|data|exports|dump)/ {
 		include snippets/off.cors-headers.include;
 		include snippets/expiry-headers.include;
-		# First attempt to serve request as file, then
-		# as directory, then fall back to displaying a 404.
-		try_files $uri $uri/ =404;
+		# First attempt to serve request from resource, then as file,
+		# then as directory, then fall back to displaying a 404.
+		try_files resources/$uri $uri $uri/ =404;
+		gzip_static always;
+		gunzip on;
+	}
+	location ~ ^/(images|fonts|css|js|donate|resources)/ {
+		include snippets/off.cors-headers.include;
+		include snippets/expiry-headers.include;
+		# First attempt to serve request as file, off_web_html acting as an override,
+		# then as directory, then fall back to displaying a 404.
+		try_files /off_web_html$uri $uri $uri/ =404;
+		gzip_static always;
+		gunzip on;
+	}
+
+	# Redirect to the change password form
+	location = /.well-known/change-password {
+		return 307 https://$host/cgi/change_password.pl;
 	}
 
 	# GoogleAssociationService made 2500 requests/min to assetlinks.json
 	# and much less when caching headers are sent
 	location = /.well-known/assetlinks.json {
-			include snippets/off.cors-headers.include;
+		include snippets/off.cors-headers.include;
 		include snippets/expiry-headers.include;
-			expires 1d;
-			try_files $uri $uri/ =404;
+		try_files $uri =404;
 	}
 
+	include snippets/obf.locations-redirects.include;
+
+	# Dynamically generated files and CGI scripts are passed
+	# to the Apache + mod_perl server running on a different
+	# port than port 80 (e.g. 8001)
+
 	location / {
 		proxy_set_header Host $host;
-		proxy_set_header X-Real-IP $remote_addr;
-		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		# recursive hosts as we are proxying behind a proxy
+		set_real_ip_from 10.0.0.0/8;
+		real_ip_recursive on;
+		access_log /var/log/nginx/proxy-obf-access.log proxied_requests buffer=256K flush=1s;
+		error_log /var/log/nginx/proxy-obf-error.log;
+
 		proxy_pass http://127.0.0.1:8002/cgi/display.pl?;
 	}
 
 	location /cgi/ {
 		proxy_set_header Host $host;
-		proxy_set_header X-Real-IP $remote_addr;
-		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		# recursive hosts as we are proxying behind a proxy
+		set_real_ip_from 10.0.0.0/8;
+		real_ip_recursive on;
+		access_log /var/log/nginx/proxy-obf-access.log proxied_requests buffer=256K flush=1s;
+		error_log /var/log/nginx/proxy-obf-error.log;
+
 		proxy_pass http://127.0.0.1:8002;
 	}
-
-	# deny access to .htaccess files, if Apache's document root
-	# concurs with nginx's one
-	#
-	#location ~ /\.ht {
-	#       deny all;
-	#}
-
 }
+

conf/nginx/snippets/obf.domain-redirects.include

@@ -0,0 +1,2 @@
+# some redirects for obf subdomains that are shortcuts to other places
+# currently no subdomains shortcuts are used

conf/nginx/snippets/obf.locations-redirects.include

@@ -0,0 +1 @@
+# Redirects for Open Beauty Facts

conf/obf-log.conf

@@ -0,0 +1,19 @@
+log4perl.rootLogger=ERROR, LOGFILE
+log4perl.logger.mongodb=INFO, MONGODB_LOGFILE
+
+log4perl.PatternLayout.cspec.S = sub { my $context = Log::Log4perl::MDC->get_context; use Data::Dumper (); local $Data::Dumper::Indent    = 0; local $Data::Dumper::Terse     = 1; local $Data::Dumper::Sortkeys  = 1; local $Data::Dumper::Quotekeys = 0; local $Data::Dumper::Deparse   = 1; my $str = Data::Dumper::Dumper($context); $str =~ s/[\n\r]/ /g; return $str; }
+
+log4perl.appender.LOGFILE=Log::Log4perl::Appender::File
+log4perl.appender.LOGFILE.filename=/srv/obf/logs/log4perl.log
+log4perl.appender.LOGFILE.mode=append
+
+log4perl.appender.LOGFILE.layout=PatternLayout
+log4perl.appender.LOGFILE.layout.ConversionPattern=[%r] %F %L %c %S %m{chomp}%n
+
+log4perl.appender.MONGODB_LOGFILE=Log::Log4perl::Appender::File
+log4perl.appender.MONGODB_LOGFILE.filename=/srv/obf/logs/mongodb_log4perl.log
+log4perl.appender.MONGODB_LOGFILE.mode=append
+
+log4perl.appender.MONGODB_LOGFILE.layout=PatternLayout
+log4perl.appender.MONGODB_LOGFILE.layout.ConversionPattern=[%r] %F %L %c %S %m{chomp}%n
+

conf/obf-minion_log.conf

@@ -0,0 +1,10 @@
+log4perl.rootLogger=DEBUG, LOGFILE
+
+log4perl.PatternLayout.cspec.S = sub { my $context = Log::Log4perl::MDC->get_context; use Data::Dumper (); local $Data::Dumper::Indent    = 0; local $Data::Dumper::Terse     = 1; local $Data::Dumper::Sortkeys  = 1; local $Data::Dumper::Quotekeys = 0; local $Data::Dumper::Deparse   = 1; my $str = Data::Dumper::Dumper($context); $str =~ s/[\n\r]/ /g; return $str; }
+
+log4perl.appender.LOGFILE=Log::Log4perl::Appender::File
+log4perl.appender.LOGFILE.filename=/srv/off-pro/logs/minion_log4perl.log
+log4perl.appender.LOGFILE.mode=append
+
+log4perl.appender.LOGFILE.layout=PatternLayout
+log4perl.appender.LOGFILE.layout.ConversionPattern=[%r] %F %L %c %S %m{chomp}%n