Add portable python 3.8

centos/Dockerfile

@@ -319,101 +319,96 @@ RUN yum install -y \
     cd .. && \
     rm -rf opencv-${OpenCV_VERSION}* opencv_contrib-${OpenCV_VERSION}*
 
-# MXNet
-ENV MXNet_DIR=/opt/mxnet-1.6.0
-ADD patches/mxnet-1.6.0-manual-cuda-arch-flags.patch /opt/mxnet-1.6.0-manual-cuda-arch-flags.patch
-RUN version=1.6.0 && \
-    source /etc/profile && \
-    source scl_source enable devtoolset-8 && \
-    cd /opt && \
-    curl -sSLO https://github.com/apache/incubator-mxnet/releases/download/${version}/apache-mxnet-src-${version}-incubating.tar.gz && \
-    tar -xf apache-mxnet-src-${version}-incubating.tar.gz && \
-    rm -f apache-mxnet-src-${version}-incubating.tar.gz && \
-    mv apache-mxnet-src-${version}-incubating mxnet-${version} && \
-    cd mxnet-${version} && \
-    patch -p1 -i /opt/mxnet-1.6.0-manual-cuda-arch-flags.patch && \
-    cmake  -S. -B build -G Ninja \
-        -D CMAKE_CXX_STANDARD=14 -D USE_CXX14_IF_AVAILABLE=ON \
-        -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=/usr/local \
-        -D CMAKE_CUDA_COMPILER=/usr/local/cuda-10.1/bin/nvcc \
-        -D USE_CUDA=ON -D USE_CUDNN=ON -D USE_NCCL=OFF \
-        -D CUDA_ARCH_FLAGS="-gencode;arch=compute_61,code=sm_61;-gencode;arch=compute_30,code=compute_30;-gencode;arch=compute_52,code=compute_52;-gencode;arch=compute_75,code=compute_75" \
-        -D USE_OPENCV=ON -D USE_OPENMP=ON \
-        -D USE_LAPACK=ON -D USE_MKL_IF_AVAILABLE=OFF \
-        -D USE_DIST_KVSTORE=OFF \
-        -D USE_CPP_PACKAGE=ON \
-        -D USE_SIGNAL_HANDLER=ON \
-        -D ENABLE_CUDA_RTC=ON \
-        -D BUILD_CPP_EXAMPLES=OFF \
-        -D USE_GPERFTOOLS=OFF \
-        -D USE_JEMALLOC=OFF \
-        && \
-    rm /opt/mxnet-1.6.0-manual-cuda-arch-flags.patch && \
-    ninja -C build
-
-# pybind11
-ARG pybind11_VERSION=2.7.1
-RUN source /etc/profile && \
-    cd /tmp && \
-    curl -sSL -o pybind11-${pybind11_VERSION}.tar.gz https://github.com/pybind/pybind11/archive/v${pybind11_VERSION}.tar.gz && \
-    tar -xf pybind11-${pybind11_VERSION}.tar.gz && \
-    cd pybind11-${pybind11_VERSION} && \
-    cmake -S. -B build -G Ninja \
-        -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \
-        -DPYBIND11_TEST=off \
-        && \
-    ninja -C build install && \
-    cd .. && \
-    rm -rf pybind11-${pybind11_VERSION}*
-
-# pylon (Basler cameras)
-ARG pylon_VERSION=5.2.0.13457
-RUN cd /tmp && \
-    curl -sSLO https://www.baslerweb.com/fp-1551786516/media/downloads/software/pylon_software/pylon-${pylon_VERSION}-x86_64.tar.gz && \
-    tar -xf pylon-${pylon_VERSION}-x86_64.tar.gz && \
-    cd pylon-${pylon_VERSION}-x86_64 && \
-    tar -xf pylonSDK-${pylon_VERSION}-x86_64.tar.gz && \
-    mv pylon5 /opt/pylon5 && \
-    cd .. && \
-    rm -rf pylon-${pylon_VERSION}*
-ADD bash/pylon.sh /etc/profile.d/pylon.sh
-
-# CGAL
-ARG CGAL_VERSION=5.2
-RUN yum install -y \
-        gmp-devel \
-        mpfr-devel \
-        && \
-    yum clean all && \
-    source /etc/profile && \
-    cd /tmp && \
-    curl -sSLO https://github.com/CGAL/cgal/releases/download/v${CGAL_VERSION}/CGAL-${CGAL_VERSION}.tar.xz && \
-    tar -xf CGAL-${CGAL_VERSION}.tar.xz && \
-    cd CGAL-${CGAL_VERSION} && \
-    cmake -S. -B build -G Ninja \
-        -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \
-        -DCGAL_INSTALL_LIB_DIR=lib -DCGAL_INSTALL_CMAKE_DIR="lib/cmake/CGAL" \
-        -DWITH_CGAL_Qt5=ON \
-        && \
-    ninja -C build install && \
-    cd .. && \
-    rm -rf CGAL-${CGAL_VERSION}*
-
-# glm
-ARG glm_VERSION=0.9.9.8
-RUN cd /tmp && \
-    curl -sSLO https://github.com/g-truc/glm/releases/download/${glm_VERSION}/glm-${glm_VERSION}.zip && \
-    unzip glm-${glm_VERSION}.zip && \
-    mv glm /opt/glm && \
-    rm -rf glm* && \
-    echo "export glm_DIR=/opt/glm/cmake/glm" > /etc/profile.d/glm.sh
-
-# Additional development dependencies
-RUN yum install -y \
-        libepoxy-devel \
-        && \
-    yum clean all && \
-    python3.6 -m pip --no-cache-dir install -U pip && \
+# # MXNet
+# ENV MXNet_DIR=/opt/mxnet-1.6.0
+# ADD patches/mxnet-1.6.0-manual-cuda-arch-flags.patch /opt/mxnet-1.6.0-manual-cuda-arch-flags.patch
+# RUN version=1.6.0 && \
+#     source /etc/profile && \
+#     source scl_source enable devtoolset-8 && \
+#     cd /opt && \
+#     curl -sSLO https://github.com/apache/incubator-mxnet/releases/download/${version}/apache-mxnet-src-${version}-incubating.tar.gz && \
+#     tar -xf apache-mxnet-src-${version}-incubating.tar.gz && \
+#     rm -f apache-mxnet-src-${version}-incubating.tar.gz && \
+#     mv apache-mxnet-src-${version}-incubating mxnet-${version} && \
+#     cd mxnet-${version} && \
+#     patch -p1 -i /opt/mxnet-1.6.0-manual-cuda-arch-flags.patch && \
+#     cmake  -S. -B build -G Ninja \
+#         -D CMAKE_CXX_STANDARD=14 -D USE_CXX14_IF_AVAILABLE=ON \
+#         -D CMAKE_BUILD_TYPE=Release -D CMAKE_INSTALL_PREFIX=/usr/local \
+#         -D CMAKE_CUDA_COMPILER=/usr/local/cuda-10.1/bin/nvcc \
+#         -D USE_CUDA=ON -D USE_CUDNN=ON -D USE_NCCL=OFF \
+#         -D CUDA_ARCH_FLAGS="-gencode;arch=compute_61,code=sm_61;-gencode;arch=compute_30,code=compute_30;-gencode;arch=compute_52,code=compute_52;-gencode;arch=compute_75,code=compute_75" \
+#         -D USE_OPENCV=ON -D USE_OPENMP=ON \
+#         -D USE_LAPACK=ON -D USE_MKL_IF_AVAILABLE=OFF \
+#         -D USE_DIST_KVSTORE=OFF \
+#         -D USE_CPP_PACKAGE=ON \
+#         -D USE_SIGNAL_HANDLER=ON \
+#         -D ENABLE_CUDA_RTC=ON \
+#         -D BUILD_CPP_EXAMPLES=OFF \
+#         -D USE_GPERFTOOLS=OFF \
+#         -D USE_JEMALLOC=OFF \
+#         && \
+#     rm /opt/mxnet-1.6.0-manual-cuda-arch-flags.patch && \
+#     ninja -C build
+
+# # pybind11
+# ARG pybind11_VERSION=2.7.1
+# RUN source /etc/profile && \
+#     cd /tmp && \
+#     curl -sSL -o pybind11-${pybind11_VERSION}.tar.gz https://github.com/pybind/pybind11/archive/v${pybind11_VERSION}.tar.gz && \
+#     tar -xf pybind11-${pybind11_VERSION}.tar.gz && \
+#     cd pybind11-${pybind11_VERSION} && \
+#     cmake -S. -B build -G Ninja \
+#         -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \
+#         -DPYBIND11_TEST=off \
+#         && \
+#     ninja -C build install && \
+#     cd .. && \
+#     rm -rf pybind11-${pybind11_VERSION}*
+
+# # pylon (Basler cameras)
+# ARG pylon_VERSION=5.2.0.13457
+# RUN cd /tmp && \
+#     curl -sSLO https://www.baslerweb.com/fp-1551786516/media/downloads/software/pylon_software/pylon-${pylon_VERSION}-x86_64.tar.gz && \
+#     tar -xf pylon-${pylon_VERSION}-x86_64.tar.gz && \
+#     cd pylon-${pylon_VERSION}-x86_64 && \
+#     tar -xf pylonSDK-${pylon_VERSION}-x86_64.tar.gz && \
+#     mv pylon5 /opt/pylon5 && \
+#     cd .. && \
+#     rm -rf pylon-${pylon_VERSION}*
+# ADD bash/pylon.sh /etc/profile.d/pylon.sh
+
+# # CGAL
+# ARG CGAL_VERSION=5.2
+# RUN yum install -y \
+#         gmp-devel \
+#         mpfr-devel \
+#         && \
+#     yum clean all && \
+#     source /etc/profile && \
+#     cd /tmp && \
+#     curl -sSLO https://github.com/CGAL/cgal/releases/download/v${CGAL_VERSION}/CGAL-${CGAL_VERSION}.tar.xz && \
+#     tar -xf CGAL-${CGAL_VERSION}.tar.xz && \
+#     cd CGAL-${CGAL_VERSION} && \
+#     cmake -S. -B build -G Ninja \
+#         -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \
+#         -DCGAL_INSTALL_LIB_DIR=lib -DCGAL_INSTALL_CMAKE_DIR="lib/cmake/CGAL" \
+#         -DWITH_CGAL_Qt5=ON \
+#         && \
+#     ninja -C build install && \
+#     cd .. && \
+#     rm -rf CGAL-${CGAL_VERSION}*
+
+# # glm
+# ARG glm_VERSION=0.9.9.8
+# RUN cd /tmp && \
+#     curl -sSLO https://github.com/g-truc/glm/releases/download/${glm_VERSION}/glm-${glm_VERSION}.zip && \
+#     unzip glm-${glm_VERSION}.zip && \
+#     mv glm /opt/glm && \
+#     rm -rf glm* && \
+#     echo "export glm_DIR=/opt/glm/cmake/glm" > /etc/profile.d/glm.sh
+
+RUN python3.6 -m pip --no-cache-dir install -U pip && \
     python3.6 -m pip --no-cache-dir install 'cryptography<3.4' &&  \
     python3.6 -m pip --no-cache-dir install \
         wheel \
@@ -425,19 +420,15 @@ RUN echo -e "/usr/local/lib\n/usr/local/lib64" > /etc/ld.so.conf.d/local.conf &&
 
 # Python build dependencies
 RUN yum install -y \
-        openssl-devel \
+        openssl11-devel \
         libffi-devel \
+        readline-devel \
+        sqlite-devel \
         && \
     yum clean all
 
 # Python 3.7
-RUN version=3.7.9 && \
-    yum install -y \
-        openssl-devel \
-        libffi-devel \
-        sqlite-devel \
-        && \
-    yum clean all && \
+RUN version=3.7.12 && \
     source /etc/profile && \
     cd /tmp && \
     curl -sSLO https://www.python.org/ftp/python/${version}/Python-${version}.tgz && \
@@ -457,13 +448,7 @@ RUN version=3.7.9 && \
     ldconfig
 
 # Python 3.8
-RUN version=3.8.10 && \
-    yum install -y \
-        openssl-devel \
-        libffi-devel \
-        sqlite-devel \
-        && \
-    yum clean all && \
+RUN version=3.8.12 && \
     source /etc/profile && \
     cd /tmp && \
     curl -sSLO https://www.python.org/ftp/python/${version}/Python-${version}.tgz && \
@@ -483,13 +468,7 @@ RUN version=3.8.10 && \
     ldconfig
 
 # Python 3.9
-RUN version=3.9.1 && \
-    yum install -y \
-        openssl-devel \
-        libffi-devel \
-        sqlite-devel \
-        && \
-    yum clean all && \
+RUN version=3.9.9 && \
     source /etc/profile && \
     cd /tmp && \
     curl -sSLO https://www.python.org/ftp/python/${version}/Python-${version}.tgz && \
@@ -558,18 +537,34 @@ RUN python3.9 -m pip --no-cache-dir install -U pip && \
 
 # Additional development dependencies
 RUN yum install -y \
+        patchelf \
+        libepoxy-devel \
         json-devel \
         && \
     yum clean all
 
 # Portable Python 3.8
 
-RUN mkdir -p /opt/python && \
-    cd /opt/python && \
-    curl -sSLO https://github.com/niess/python-appimage/releases/download/python3.8/python3.8.12-cp38-cp38-manylinux1_x86_64.AppImage && \
-    chmod +x python3.8.12-cp38-cp38-manylinux1_x86_64.AppImage && \
-    ./python3.8.12-cp38-cp38-manylinux1_x86_64.AppImage --appimage-extract && \
-    rm -f python3.8.12-cp38-cp38-manylinux1_x86_64.AppImage && \
-    mv squashfs-root 3.8
+RUN version=3.8.12 && \
+    source /etc/profile && \
+    cd /tmp && \
+    curl -sSLO https://www.python.org/ftp/python/${version}/Python-${version}.tgz && \
+    tar -xf Python-${version}.tgz && \
+    cd Python-${version} && \
+    CFLAGS="-fPIC" LDFLAGS="-fPIC" ./configure \
+        --prefix=/opt/python/3.8 \
+        --enable-optimizations \
+        --enable-shared \
+        --with-ensurepip=install \
+        && \
+    make -j$(nproc) altinstall && \
+    cd .. && \
+    rm -rf Python-${version}*
+
+ADD python/make-python-portable.py /usr/local/bin/make-python-portable.py
+RUN yum install -y patchelf && \
+    yum clean all && \
+    source /etc/profile && \
+    make-python-portable.py /opt/python/3.8/bin/python3.8
 
 RUN tar -C /opt/python/3.8/ --numeric-owner --xattrs --xattrs-include='*' -cf /opt/python/python-3.8.tar .

centos/python/make-python-portable.py

+#! /usr/bin/env python3
+
+import sys
+import re
+import shutil
+
+from pathlib import Path
+from subprocess import check_output, check_call
+
+
+def parse_ldd_line(line: str):
+    line = line.strip()
+    line = re.sub(r" *\(0x[0-9a-fA-F]*\)", "", line)
+    parts = line.split("=>")
+    name = Path(parts[0].strip()).name
+    if name not in exlude_list and len(parts) == 2:
+        return name, Path(parts[1].strip()).resolve()
+
+
+def collect_dependencies(target):
+    target = Path(target)
+
+    def collect():
+        for line in check_output(["ldd", target]).decode("utf-8").splitlines():
+            dep = parse_ldd_line(line)
+            if dep is not None:
+                yield dep
+
+    return list(collect())
+
+def make_python_portable(interpreter):
+    interpreter = Path(interpreter)
+    bin_dir = interpreter.parent
+    root_dir = bin_dir.parent
+    lib_dir = root_dir / "lib"
+
+    # Install all dependencies
+    targets = [
+        interpreter,
+        *list(Path(lib_dir / (interpreter.resolve().name)).glob("**/*.so"))
+    ]
+
+    deps = []
+    for target in targets:
+        deps += collect_dependencies(target)
+    for name, path in deps:
+        if not root_dir in path.parents:
+            shutil.copy2(str(path), str(lib_dir / name))
+
+    # Patch targets and dependencies to find each other
+    check_call(["patchelf", "--set-rpath", "$ORIGIN/../lib", interpreter])
+    for target in targets[1:]:
+        num_parents = len(target.relative_to(lib_dir).parts) - 1
+        rpath = Path("$ORIGIN")
+        for _ in range(num_parents):
+            rpath /= ".."
+        check_call(["patchelf", "--set-rpath", str(rpath), target])
+    for name, path in deps:
+        check_call(["patchelf", "--set-rpath", "$ORIGIN", str(lib_dir / name)])
+
+# Retrieved: 2021-11-18
+appimage_exclude_list = [
+    "ld-linux.so.2",
+    "ld-linux-x86-64.so.2",
+    "libanl.so.1",
+    "libBrokenLocale.so.1",
+    "libcidn.so.1",
+    "libc.so.6",
+    "libdl.so.2",
+    "libm.so.6",
+    "libmvec.so.1",
+    "libnss_compat.so.2",
+    "libnss_dns.so.2",
+    "libnss_files.so.2",
+    "libnss_hesiod.so.2",
+    "libnss_nisplus.so.2",
+    "libnss_nis.so.2",
+    "libpthread.so.0",
+    "libresolv.so.2",
+    "librt.so.1",
+    "libthread_db.so.1",
+    "libutil.so.1",
+    "libstdc++.so.6",
+    "libGL.so.1",
+    "libEGL.so.1",
+    "libGLdispatch.so.0",
+    "libGLX.so.0",
+    "libOpenGL.so.0",
+    "libdrm.so.2",
+    "libglapi.so.0",
+    "libgbm.so.1",
+    "libxcb.so.1",
+    "libX11.so.6",
+    "libgio-2.0.so.0",
+    "libasound.so.2",
+    "libfontconfig.so.1",
+    "libthai.so.0",
+    "libfreetype.so.6",
+    "libharfbuzz.so.0",
+    "libcom_err.so.2",
+    "libexpat.so.1",
+    "libgcc_s.so.1",
+    "libglib-2.0.so.0",
+    "libgpg-error.so.0",
+    "libICE.so.6",
+    "libp11-kit.so.0",
+    "libSM.so.6",
+    "libusb-1.0.so.0",
+    "libuuid.so.1",
+    "libz.so.1",
+    "libgobject-2.0.so.0",
+    "libpangoft2-1.0.so.0",
+    "libpangocairo-1.0.so.0",
+    "libpango-1.0.so.0",
+    "libgpg-error.so.0",
+    "libjack.so.0",
+    "libxcb-dri3.so.0",
+    "libxcb-dri2.so.0",
+    "libfribidi.so.0",
+    "libgmp.so.10",
+]
+
+exlude_list = [
+    *appimage_exclude_list,
+    "linux-vdso.so.1",
+]
+
+if __name__ == "__main__":
+    if len(sys.argv) >= 2:
+        make_python_portable(sys.argv[1])
+    else:
+        raise SystemExit(1)