Skip to content

Commit

Permalink
chore: two apache fixes after putting in production (#11230)
Browse files Browse the repository at this point in the history
There were some surprises as we put in production, so here are the fix.

There are also additional files that were modified directly in
production.
  • Loading branch information
alexgarel authored Jan 10, 2025
1 parent b720f67 commit a7b1e7d
Show file tree
Hide file tree
Showing 7 changed files with 72 additions and 235 deletions.
16 changes: 8 additions & 8 deletions conf/nginx/sites-available/off
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@ map $uri $apache_port {
default 8001;

# home pages
"~*/$" 8002;
"~*^/$" 8002;
# product read / write (note that nginx does not support direct utf-8)
"~*/(mountaj|m\xc9\x99hsul|\xd0\xbf\xd1\x80\xd0\xbe\xd0\xb4\xd1\x83\xd0\xba\xd1\x82|gynnyrch|produkt|product|product|product|produkto|producto|toode|produkto|produit|produto|term\xc3\xa9k|produk|\xe8\xa3\xbd\xe5\x93\x81|afaris|\xd3\xa9\xd0\xbd\xd1\x96\xd0\xbc|\xec\x83\x9d\xec\x84\xb1\xeb\xac\xbc|berhem|\xe0\xa4\x89\xe0\xa4\xa4\xe0\xa5\x8d\xe0\xa4\xaa\xe0\xa4\xbe\xe0\xa4\xa6\xe0\xa4\xa8|produk|produkt|\xe0\xa4\x89\xe0\xa4\xa4\xe0\xa5\x8d\xe0\xa4\xaa\xe0\xa4\xbe\xe0\xa4\xa6\xe0\xa4\xa8|product|product|product|produkt|produkt|produit|produto|produto|produto|\xd0\xbf\xd1\x80\xd0\xbe\xd0\xb4\xd1\x83\xd0\xba\xd1\x82|product|proizvod|produkto|\xc3\xbcr\xc3\xbcn|\xd0\xbf\xd1\x80\xd0\xbe\xd0\xb4\xd1\x83\xd0\xba\xd1\x82|\xe4\xba\xa7\xe5\x93\x81|\xe7\x94\xa2\xe5\x93\x81|\xe7\x94\xa2\xe5\x93\x81)/" 8002;
"~*/cgi/product.pl" 8002;
"~*^/(mountaj|m\xc9\x99hsul|\xd0\xbf\xd1\x80\xd0\xbe\xd0\xb4\xd1\x83\xd0\xba\xd1\x82|gynnyrch|produkt|product|product|product|produkto|producto|toode|produkto|produit|produto|term\xc3\xa9k|produk|\xe8\xa3\xbd\xe5\x93\x81|afaris|\xd3\xa9\xd0\xbd\xd1\x96\xd0\xbc|\xec\x83\x9d\xec\x84\xb1\xeb\xac\xbc|berhem|\xe0\xa4\x89\xe0\xa4\xa4\xe0\xa5\x8d\xe0\xa4\xaa\xe0\xa4\xbe\xe0\xa4\xa6\xe0\xa4\xa8|produk|produkt|\xe0\xa4\x89\xe0\xa4\xa4\xe0\xa5\x8d\xe0\xa4\xaa\xe0\xa4\xbe\xe0\xa4\xa6\xe0\xa4\xa8|product|product|product|produkt|produkt|produit|produto|produto|produto|\xd0\xbf\xd1\x80\xd0\xbe\xd0\xb4\xd1\x83\xd0\xba\xd1\x82|product|proizvod|produkto|\xc3\xbcr\xc3\xbcn|\xd0\xbf\xd1\x80\xd0\xbe\xd0\xb4\xd1\x83\xd0\xba\xd1\x82|\xe4\xba\xa7\xe5\x93\x81|\xe7\x94\xa2\xe5\x93\x81|\xe7\x94\xa2\xe5\x93\x81)/" 8002;
"~*^/cgi/product.pl" 8002;
# product API read / write
"~*/api/v./product/" 8002;
"~*^/api/v./product/" 8002;
}

# variables definitions for expiry headers are loaded from /etc/nginx/conf.d/expires-no-json-xml.conf
Expand Down Expand Up @@ -160,8 +160,8 @@ server {
real_ip_recursive on;
access_log /var/log/nginx/proxy-off-access.log proxied_requests buffer=256K flush=1s;
error_log /var/log/nginx/proxy-off-error.log;

proxy_pass http://127.0.0.1:$apache_port/cgi/display.pl?;
# Note: as soon as we use a variable in proxy_pass, $uri and the rest must be explicit
proxy_pass http://127.0.0.1:$apache_port/cgi/display.pl?$uri$is_args$args;
}

location /cgi/ {
Expand All @@ -171,8 +171,8 @@ server {
real_ip_recursive on;
access_log /var/log/nginx/proxy-off-access.log proxied_requests buffer=256K flush=1s;
error_log /var/log/nginx/proxy-off-error.log;

proxy_pass http://127.0.0.1:$apache_port;
# Note: as soon as we use a variable in proxy_pass, $uri and the rest must be explicit
proxy_pass http://127.0.0.1:$apache_port$uri$is_args$args;
}
}

Large diffs are not rendered by default.

12 changes: 10 additions & 2 deletions docs/dev/how-to-release.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@ corresponding to the service short name (off, opf, etc.) and the version tag.
To deploy you need to execute the following steps:
1. merge the Release Please pull request.
This will create a new release / version tag on github
1. verify there is no unreleased code on the server:
```bash
sudo -u off bash
cd /srv/$SERVICE
git status
```
1. update the code:
```bash
sudo -u off bash
Expand All @@ -32,7 +38,7 @@ To deploy you need to execute the following steps:
```
1. verify every needed symlink is in place
```bash
sudo -u off scripts/deploy/verify-deployment.sh $SERVICE
sudo /srv/$SERVICE/scripts/deploy/verify-deployment.sh $SERVICE
```
1. rebuild taxonomies and lang
```bash
Expand All @@ -50,6 +56,8 @@ To deploy you need to execute the following steps:
```bash
sudo systemctl daemon-reload
sudo systemctl restart nginx
sudo systemctl stop apache2 cloud_vision_ocr@$SERVICE.service minion@$SERVICE.service
sudo systemctl stop apache2 cloud_vision_ocr@$SERVICE.service minion@$SERVICE.service; \
sudo systemctl start apache2 cloud_vision_ocr@$SERVICE.service minion@$SERVICE.service
# On off
sudo systemctl stop apache2@priority; sudo systemctl start apache2@priority
```
186 changes: 1 addition & 185 deletions html/donate/fr.helloasso.html
Original file line number Diff line number Diff line change
Expand Up @@ -48,194 +48,10 @@
<meta name="apple-itunes-app" content="app-id=588797948" />
<meta name="flattr:id" content="dw637l" />

<link
rel="stylesheet"
href="/cgi/css.pl"
/>
<link
rel="stylesheet"
href="https://static.openfoodfacts.org/css/dist/jqueryui/themes/base/jquery-ui.css"
/>
<link
rel="stylesheet"
href="https://cdnjs.cloudflare.com/ajax/libs/select2/4.0.3/css/select2.min.css"
integrity="sha384-HIipfSYbpCkh5/1V87AWAeR5SUrNiewznrUrtNz1ux4uneLhsAKzv/0FnMbj3m6g"
crossorigin="anonymous"
/>
<link
rel="search"
href="https://en.openfoodfacts.org/cgi/opensearch.pl"
type="application/opensearchdescription+xml"
title="Open Food Facts"
/>
<style media="all">
.show-when-logged-in {
display: none;
}
.reason img {
height: 100px;
}
p {
line-height: 1.3;
margin-bottom: 0.8rem;
}
.wrapper {
background-image: url("https://static.openfoodfacts.org/images/misc/2023-donation/background.png");
background-size: contain;/**/
background-repeat: no-repeat;/**/
background-size: 100% 100%;
z-index: 0;
}
</style>
<script src="https://static.openfoodfacts.org/js/dist/jquery.js"></script>
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(["_setAccount", "UA-31851927-1"]);
_gaq.push(["_setDomainName", "openfoodfacts.org"]);
_gaq.push(["_trackPageview"]);

(function() {
var ga = document.createElement("script");
ga.type = "text/javascript";
ga.async = true;
ga.src =
("https:" == document.location.protocol
? "https://ssl"
: "http://www") + ".google-analytics.com/ga.js";
var s = document.getElementsByTagName("script")[0];
s.parentNode.insertBefore(ga, s);
})();

$(function() {

$('iframe').load(function(){
$(".loading").remove();
}).show();

});

</script>
</head>
<body>
<div class="wrapper">
<div class="hide-for-large-up text-center">
<img
src="https://static.openfoodfacts.org/images/logos/off-logo-horizontal-light.svg"
width="178"
height="156"
alt="Logo Open Food Facts"
/>
</div>

<div class="hide-for-medium hide-for-large-up text-center">
<div class="panel callout" style="padding-top: 1rem; padding-bottom: 1rem; margin-bottom: 0.4rem;">
<h3>Soutenez Open Food Facts pour 2024 !</h3>
<p>
Open Food Facts est 100% gratuit et indépendant de l'industrie alimentaire.
Nous avons besoin de votre aide pour continuer à développer le projet.
</p>
</div>
</div>

<div
class="row"
style="width: 100%; margin-left: auto; margin-right: auto; max-width: initial; margin-top:10px;"
>
<div class="columns large-2 hide-for-medium hide-for-small text-center">
<img
src="https://static.openfoodfacts.org/images/logos/off-logo-horizontal-light.svg"
width="178"
height="156"
alt="Logo Open Food Facts"
/>
</div>

<div
class="columns medium-6 large-5 xlarge-6 xxlarge-7 hide-for-small"
>

<div class="panel callout" style="padding-top: 1rem; padding-bottom: 1rem; margin-bottom: 0.8rem;">
<h3>Soutenez Open Food Facts pour 2024 !</h3>
<p>
Open Food Facts est 100% gratuit et indépendant de l'industrie alimentaire.
Nous avons besoin de votre aide pour continuer à développer le projet.
</p>
</div>

<h3>Pourquoi donner à Open Food Facts ?</h3>
<p>
Open Food Facts est le "Wikipedia de l’alimentation", la plus grande base de données libre et ouverte de produits alimentaires du monde entier.
Nous avons commencé Open Food Facts parce que nous croyons que les informations sur ce que
nous mangeons sont d'intérêt public et doivent
être publiques. gratuites et ouvertes.
</p>

<div class="reasons row">
<div class="reason small-12 large-6 columns">
<div class="flex-video">
<iframe width="420" height="315" src="https://www.youtube.com/embed/D1jzT02IBRA"
title="YouTube video player" frameborder="0"
allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
allowfullscreen></iframe>
</div>
</div>
<div class="reason small-12 large-6 columns">
<p>
Lorsque nous avons fondé Open Food Facts, nous avons ouvertement choisi d'en faire une
<b>association à but non-lucratif</b> pour maximiser notre impact, grâce à la gratuité de nos travaux.
Nous avons aussi pris la décision sans appel d'être <b>totalement indépendants
de l'industrie agro-alimentaire</b>, et de refuser tout financement qui pourrait nous biaiser et
corrompre notre mission.
</p>
<p>
Nous avons besoin de votre aide pour continuer à faire fonctionner et à développer Open Food Facts.
Merci d'envisager de faire un don pour protéger et soutenir Open Food Facts.
</p>
<p>Merci à vous,<br/>
Anca, Éloïse, Florence, Léonore, Marie, Christian, Ludovic,
Sébastien, Vincent et toute l'équipe Open Food Facts.
</p>
</div>
</div>
<div class="reasons row">
<div class="reason small-12 large-6 columns">
<h3>À quoi servent vos dons</h3>

<p>
<strong>Technologie :</strong> Développement, maintenance régulière et serveurs pour suivre la
croissance de la base de données et ajouter de chouettes nouvelles fonctionnalités à
l'application mobile et au site web Open Food Facts.
</p>

<p>
<strong>Volontaires et projets :</strong> Pour soutenir au mieux la communauté Open Food Facts,
pour collaborer avec des chercheurs, et pour que nos données alimentaires aient le plus
grand impact possible sur notre santé, notre planète et notre société.
</p>
</div>
<div class="small-12 large-6 columns">
<img src="https://static.openfoodfacts.org/images/misc/group-photo-small.jpg"
alt="Last meeting in Paris"
>
</div>
</div>


</div>

<div class="columns small-12 medium-6 large-5 xlarge-4 xxlarge-3 text-center">
<div style="max-width:425px">
<div class="loading">
<p>Le formulaire de don est en cours de chargement.<br>Vous pouvez aussi y accéder en <a href="https://www.helloasso.com/associations/open-food-facts/formulaires/1">cliquant ici</a>.</p>
</div>

<iframe id="haWidget" allowtransparency="true" scrolling="auto"
src="https://www.helloasso.com/associations/open-food-facts/formulaires/1/widget"
style="width:100%;height:750px;border:none;"></iframe>
</div>
</div>
</div>
</div>
<iframe id="iframe_assoconnect" src="https://open-food-facts.assoconnect.com/collect/description/476750-c-faire-un-don-a-open-food-facts?iframe=1" width="100%" style="overflow: hidden; border: 0; max-height: none;" scrolling="no" onload="window.location.href='#iframe_assoconnect'"></iframe><script>window.addEventListener("message", function(event) {if(event.data.action === "iframe.height" && (event.origin === "https://open-food-facts.assoconnect.com" || event.origin === "https://pay.assoconnect.com")){document.getElementById("iframe_assoconnect").height = event.data.height;}});</script><style>#iframe_assoconnect{border: 0}</style>

</body>
</html>
20 changes: 9 additions & 11 deletions scripts/deploy/verify-deployment.sh
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ function check_args {
exit -2
fi
ZFS_PATH="/mnt/$SERVICE"
OFF_ZFS_PATH="/mnt/off"
IS_PRO=$(is_pro "$SERVICE")
NON_PRO_SERVICE=$(non_pro_name "$SERVICE")
PRO_SERVICE=$NON_PRO_SERVICE"-pro"
Expand All @@ -79,15 +80,15 @@ function check_args {

function compute_services {
# systemd services to check for symlinks
SYSTEMD_LINKS+=( [email protected] nginx.service.d apache2.service [email protected] [email protected].d [email protected] )
SYSTEMD_LINKS+=( [email protected] nginx.service.d apache2.service.d [email protected] )
# units that must be active (and enabled)
SYSTEMD_UNITS_ACTIVE=( nginx.service apache2.service cloud_vision_ocr@$SERVICE.service )
# units that must be enabled
SYSTEMD_UNITS_ENABLED=( )
# priority request on off
if [[ $SERVICE = "off" ]]
then
SYSTEMD_LINKS+=( [email protected] )
SYSTEMD_LINKS+=( [email protected] [email protected] )
SYSTEMD_UNITS_ACTIVE+=( [email protected] prometheus-apache-exporter.service [email protected] )
fi
if [[ -z $IS_PRO ]]
Expand All @@ -111,7 +112,6 @@ function compute_expected_links {
EXPECTED_LINKS["$REPO_PATH/log.conf"]="$REPO_PATH/conf/$SERVICE-log.conf"
EXPECTED_LINKS["$REPO_PATH/minion_log.conf"]="$REPO_PATH/conf/$SERVICE-minion_log.conf"
# config
EXPECTED_LINKS["$REPO_PATH/lib/ProductOpener/Config.pm"]="$REPO_PATH/lib/ProductOpener/Config_$NON_PRO_SERVICE.pm"
EXPECTED_LINKS["$REPO_PATH/po/site-specific"]="$REPO_PATH/po/$SERVICE_LONG_NAME"
# off-web
EXPECTED_LINKS["$REPO_PATH/lang"]="/srv/openfoodfacts-web/lang"
Expand All @@ -120,8 +120,9 @@ function compute_expected_links {
EXPECTED_LINKS["$REPO_PATH/data"]="$ZFS_PATH/data"
EXPECTED_LINKS["$REPO_PATH/orgs"]="$ZFS_PATH/orgs"
EXPECTED_LINKS["$REPO_PATH/users"]="$ZFS_PATH/users"
EXPECTED_LINKS["$REPO_PATH/products"]="$ZFS_PATH/products"
EXPECTED_LINKS["$REPO_PATH/html/images/products"]="$ZFS_PATH/images/products"
# image and products are now merges on off zfs storage
EXPECTED_LINKS["$REPO_PATH/products"]="$OFF_ZFS_PATH/products"
EXPECTED_LINKS["$REPO_PATH/html/images/products"]="$OFF_ZFS_PATH/images/products"
# public data
EXPECTED_LINKS["$REPO_PATH/html/data"]="$ZFS_PATH/html_data"
EXPECTED_LINKS["$REPO_PATH/html/exports"]="$ZFS_PATH/html_data/exports"
Expand Down Expand Up @@ -203,12 +204,9 @@ function compute_expected_links {
EXPECTED_LINKS["/etc/apache2/off-envvars"]="$REPO_PATH/conf/apache-2.4/off-envvars"
EXPECTED_LINKS["/etc/apache2/mods-available/mpm_prefork.conf"]="$REPO_PATH/conf/apache-2.4/mpm_prefork.conf"
EXPECTED_LINKS["/etc/apache2/sites-enabled/$SERVICE.conf"]="$REPO_PATH/conf/apache-2.4/sites-available/$SERVICE.conf"
EXPECTED_LINKS["/etc/apache2-standard"]="/etc/apache-2"
EXPECTED_LINKS["/var/log/apache2-standard"]="/var/log/apache-2"
if [[ $SERVICE = "off" ]]
then
EXPECTED_LINKS["/etc/apache2-priority"]="/etc/apache-2"
EXPECTED_LINKS["/var/log/apache2-standard"]="/var/log/apache-2"
EXPECTED_LINKS["/etc/apache2-priority"]="/etc/apache2"
fi

for systemd_unit in {apache2,nginx}.service.d ${SYSTEMD_LINKS[@]}
Expand Down Expand Up @@ -244,11 +242,11 @@ function check_links {
GOT_ERROR=1
if [[ ! -e $target ]]
then
>&2 echo "ERROR: link $target does not exist"
>&2 echo "ERROR: link $target does not exist (should link to $destination)"
else
if [[ ! -e $destination ]]
then
>&2 echo "ERROR: link $destination does not exist"
>&2 echo "ERROR: link $destination does not exist (while $target links to it)"
else
>&2 echo "ERROR: link instead of $target -> $destination, got $(readlink -f $target) instead"
fi
Expand Down
10 changes: 9 additions & 1 deletion scripts/gen_users_emails.pl
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,22 @@

my $emails_ref = retrieve("$BASE_DIRS{USERS}/users_emails.sto");

foreach my $userid (@userids) {
my $i = 0;
my $n = scalar @userids;

foreach my $userid (sort @userids) {
my $user_ref = retrieve_user($userid);
if (defined $user_ref) {
my $email = $user_ref->{email};
if ((defined $email) and ($email =~ /\@/)) {
$emails_ref->{$email} = [$userid];
}
}
$i++;
if ($i % 1000 == 0) {
print "$i / $n - $userid\n";
store("$BASE_DIRS{USERS}/users_emails.sto", $emails_ref);
}
}

store("$BASE_DIRS{USERS}/users_emails.sto", $emails_ref);
Expand Down
Loading

0 comments on commit a7b1e7d

Please sign in to comment.