[Bug]: Build scripts are not idempotent

[whg517]

Discussed in #46880

^{Originally posted by whg517 January 13, 2025}

env

• os: registry.access.redhat.com/ubi9
• ldb-toolchain: 0.21
• java: 17
• doris: 3.0.3-rc04

build script

FROM registry.access.redhat.com/ubi9

ARG LDB_TOOLCHAIN_VERSION=0.21
ARG PRODUCT_VERSION=3.0.3-rc04
ARG JAVA_VERSION=17

# install system requirements
RUN <<EOT
    # https://adoptium.net/en-GB/installation/linux/#_centosrhelfedora_instructions
    cat <<EOF > /etc/yum.repos.d/adoptium.repo
[Adoptium]
name=Adoptium
baseurl=https://packages.adoptium.net/artifactory/rpm/${DISTRIBUTION_NAME:-$(. /etc/os-release; echo $ID)}/\$releasever/\$basearch
enabled=1
gpgcheck=1
gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public
EOF

    dnf update -y
    dnf install -y \
        automake \
        bzip2 \
        cmake \
        cyrus-sasl-devel \
        diffutils \
        fuse-devel \
        gcc \
        gcc-c++ \
        gettext \
        git \
        gzip \
        hostname \
        krb5-devel \
        libcurl-devel \
        libtool \
        make \
        maven \
        nodejs \
        openssl-devel \
        patch \
        pkg-config \
        perl-CPAN \
        tar \
        temurin-17-jdk \
        tzdata-java \
        unzip \
        wget \
        which \
        xz \
        zlib-devel \
        zip

    microdnf clean all
    rm -rf /var/cache/yum

    # smoke test
    java -version
EOT

ENV JAVA_HOME=/usr/lib/jvm/temurin-${JAVA_VERSION}-jdk
ENV JAVA_VERSION=${JAVA_VERSION}

# setup gettext
# Because of the gettext of dnf is lack autopoint, so we need to install gettext from source.
# Download gettext https://ftp.gnu.org/gnu/gettext/gettext-0.21.tar.gz
RUN <<EOF
    set -ex

    # Get already installed gettext version
    GETTEXT_VERSION=$(gettext --version | head -n 1 | awk '{print $4}')

    mkdir -p /build/gettext
    mkdir -p /opt/gettext
    pushd /build/gettext
    curl -sSLf https://ftp.gnu.org/gnu/gettext/gettext-${GETTEXT_VERSION}.tar.gz \
        | tar -xz --strip-components=1

    ./configure --prefix=/opt/gettext
    make -j$(nproc)
    make install
    popd

    # cleanup
    rm -rf /build/gettext
EOF

ENV PATH="/opt/gettext/bin:$PATH"

# setup ldb-toolchain
# Download ldb_toolchain_gen.aarch64.sh
# https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.9.1/ldb_toolchain_gen.aarch64.sh
# or https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.9.1/ldb_toolchain_gen.sh
# Note:
#   - In 0.9.1, ldb-toolchain containes curl, it conflicts with the system curl,
#     when use it, it will cause dns resolution error
#   - In 0.21, ldb-toolchain containes curl, it will verify the certificate
#     file /etc/ssl/certs/ca-certificates.crt. But ca-certificates package generate a default
#     file /etc/ssl/certs/ca-bundle.crt, so we need to create a symlink to fix it.
#   - 0.14.1 aaarch64 version is error to build doris
RUN <<EOF
    set -ex
    ARCH=$(uname -m)
    ARCH=${ARCH/arm64/aarch64}

    release_file="ldb_toolchain_gen.sh"
    if [ "$ARCH" == "aarch64" ]; then
        release_file="ldb_toolchain_gen.aarch64.sh"
    fi

    mkdir -p /build/ldb-toolchain
    pushd /build/ldb-toolchain
    curl -sSLf https://github.com/amosbird/ldb_toolchain_gen/releases/download/v${LDB_TOOLCHAIN_VERSION}/${release_file} \
        -o ldb_toolchain_gen.sh

    chmod +x ldb_toolchain_gen.sh

    ./ldb_toolchain_gen.sh /opt/ldb-toolchain
    popd

    # create a symlink to bison, because doris need byacc
    ln -s /opt/ldb-toolchain/bin/bison /opt/ldb-toolchain/bin/byacc

    # fix /opt/ldb-toolchain/bin/curl certificate verify failed
    ln -s /etc/ssl/certs/ca-bundle.crt /etc/ssl/certs/ca-certificates.crt

    # cleanup
    rm -rf /build/ldb-toolchain
EOF

ENV PATH="/opt/ldb-toolchain/bin:$PATH"

# build doris
# Download doris https://github.com/apache/doris/archive/refs/tags/3.0.3-rc04.tar.gz
RUN <<EOF
    set -ex
    # alias python3 to python
    ln -s /usr/bin/python3 /usr/bin/python

    mkdir -p /build/doris
    pushd /build/doris
    curl -sSLf https://github.com/apache/doris/archive/refs/tags/${PRODUCT_VERSION}.tar.gz \
        | tar -xz --strip-components=1

    # Add `set -x` after `set -euo pipefail`
    sed -i '/set -eo pipefail/a set -x' /build/doris/build.sh
    sed -i '/set -eo pipefail/a set -x' /build/doris/thirdparty/build-thirdparty.sh

    # build doris, disable avx2
    export USE_AVX2=OFF
    export USE_UNWIND=OFF
    # ./build.sh
    nohup ./build.sh > /build/doris-build.log 2>&1 &
    tail -f /build/doris-build.log

    # TODO
EOF

error

[799/1371] Linking CXX static library src/runtime/libRuntime.a
[800/1371] Building CXX object src/service/CMakeFiles/Service.dir/brpc_service.cpp.o
[801/1371] Building CXX object src/service/CMakeFiles/Service.dir/backend_service.cpp.o
[802/1371] Building CXX object src/service/CMakeFiles/Service.dir/http_service.cpp.o
[803/1371] Building CXX object src/service/CMakeFiles/Service.dir/point_query_executor.cpp.o
[804/1371] Building CXX object src/service/CMakeFiles/Service.dir/internal_service.cpp.o
FAILED: src/service/CMakeFiles/Service.dir/internal_service.cpp.o
/opt/ldb-toolchain/bin/clang++ -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG -DBOOST_STACKTRACE_USE_BACKTRACE -DBOOST_SYSTEM_NO_DEPRECATED -DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1 -DBRPC_ENABLE_CPU_PROFILER -DGLOG_CUSTOM_PREFIX_SUPPORT -DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H -DLIBJVM -DS2_USE_GFLAGS -DS2_USE_GLOG -DUSE_HADOOP_HDFS -DUSE_JEMALLOC -DUSE_MEM_TRACKER -D__STDC_FORMAT_MACROS -I/build/doris/be/src/apache-orc/c++/include -I/build/doris/be/build_Release/src/apache-orc/c++/include -I/build/doris/be/build_Release/src/clucene/src/shared -I/build/doris/be/src/clucene/src/core -I/build/doris/be/src/clucene/src/shared -I/build/doris/be/src/clucene/src/contribs-lib -I/build/doris/be/src -I/build/doris/be/test -I/usr/lib/jvm/temurin-17-jdk/include -I/usr/lib/jvm/temurin-17-jdk/include/linux -isystem /build/doris/be/../common -isystem /build/doris/be/../gensrc/build -isystem /build/doris/thirdparty/installed/include -isystem /build/doris/thirdparty/installed/gperftools/include -O3 -DNDEBUG  -O3 -DNDEBUG -std=gnu++20   -D OS_LINUX -g -Wall -Wextra -Werror -pthread -fstrict-aliasing -fno-omit-frame-pointer -Wnon-virtual-dtor -Wno-unused-parameter -Wno-sign-compare -fcolor-diagnostics -Wpedantic -Wshadow-field -Wunused -Wunused-command-line-argument -Wunused-exception-parameter -Wunused-volatile-lvalue -Wunused-template -Wunused-member-function -Wunused-macros -Wconversion -Wno-gnu-statement-expression -Wno-implicit-float-conversion -Wno-implicit-int-conversion -Wno-sign-conversion -Wno-missing-field-initializers -Wno-unused-const-variable -Wno-shorten-64-to-32 -march=armv8-a+crc -Winvalid-pch -Xclang -include-pch -Xclang /build/doris/be/build_Release/CMakeFiles/pch.dir/cmake_pch.hxx.pch -Xclang -include -Xclang /build/doris/be/build_Release/CMakeFiles/pch.dir/cmake_pch.hxx -MD -MT src/service/CMakeFiles/Service.dir/internal_service.cpp.o -MF src/service/CMakeFiles/Service.dir/internal_service.cpp.o.d -o src/service/CMakeFiles/Service.dir/internal_service.cpp.o -c /build/doris/be/src/service/internal_service.cpp
In file included from /build/doris/be/src/service/internal_service.cpp:134:
/build/doris/be/src/vec/exec/format/orc/vorc_reader.h:661:63: error: non-virtual member function marked 'override' hides virtual member function
  661 |                           std::vector<bool> selected_columns) override;
      |                                                               ^
/build/doris/be/src/apache-orc/c++/include/orc/OrcFile.hh:66:18: note: hidden overloaded virtual function 'orc::InputStream::beforeReadStripe' declared here: different number of parameters (3 vs 2)
   66 |     virtual void beforeReadStripe(
      |                  ^
1 error generated.
[805/1371] Building CXX object src/udf/CMakeFiles/Udf.dir/udf.cpp.o
[806/1371] Building CXX object src/service/CMakeFiles/doris_be.dir/doris_main.cpp.o
[807/1371] Building CXX object src/udf/CMakeFiles/DorisUdf.dir/udf.cpp.o
[808/1371] Building CXX object src/cloud/CMakeFiles/Cloud.dir/cloud_backend_service.cpp.o
[809/1371] Building CXX object src/olap/CMakeFiles/Olap.dir/delete_handler.cpp.o
[810/1371] Building CXX object src/olap/CMakeFiles/Olap.dir/tablet_reader.cpp.o
ninja: build stopped: subcommand failed.

files

source code from doris-thirdparty head
https://github.com/apache/doris-thirdparty/blob/f10bb3d91136149845f69fa51ad3a0663941c43d/c%2B%2B/include/orc/OrcFile.hh#L66

    virtual void beforeReadStripe(
        std::unique_ptr<StripeInformation> currentStripeInformation,
        std::vector<bool> selectedColumns,
        std::unordered_map<orc::StreamId, std::shared_ptr<InputStream>>& streams);

source code from tag 3.0.3-rc04

doris/be/src/vec/exec/format/orc/vorc_reader.h

Lines 660 to 661 in 62a58bf

	void beforeReadStripe(std::unique_ptr<orc::StripeInformation> current_strip_information,
	std::vector<bool> selected_columns) override;

desc

The build failed at tag time because the latest code was used when fetching the submodule.

doris/build.sh

Lines 552 to 555 in 62a58bf

	if [[ "${BUILD_BE}" -eq 1 ]]; then
	update_submodule "be/src/apache-orc" "apache-orc" "https://github.com/apache/doris-thirdparty/archive/refs/heads/orc.tar.gz"
	update_submodule "be/src/clucene" "clucene" "https://github.com/apache/doris-thirdparty/archive/refs/heads/clucene.tar.gz"
	if [[ -e "${DORIS_HOME}/gensrc/build/gen_cpp/version.h" ]]; then

The source code downloaded in tag does not have git commands. So when you get a submodule, you get the latest code from the submodule repository.

doris/build.sh

Lines 308 to 317 in 62a58bf

	if [[ "${exit_code}" -ne 0 ]]; then
	set +e
	# try to get submodule's current commit
	submodule_commit=$(git ls-tree HEAD "${submodule_path}" | awk '{print $3}')
	exit_code=$?
	if [[ "${exit_code}" = "0" ]]; then
	commit_specific_url=$(echo "${archive_url}" | sed "s/refs\/heads/${submodule_commit}/")
	else
	commit_specific_url="${archive_url}"
	fi