Skip to content

Upgrade Python version #257

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
+12 −116
Open

Upgrade Python version #257

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
79953ca
upgrade py version
jinyan-li1 Nov 6, 2025
08f3902
update license format
jinyan-li1 Nov 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 4 additions & 106 deletions docker/1.4-2/base/Dockerfile.cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,103 +18,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

# MLIO builder stage with Ubuntu 20.04
FROM ubuntu:20.04@sha256:874aca52f79ae5f8258faff03e10ce99ae836f6e7d2df6ecd3da5c1cad3a912b as mlio-builder

ARG PYTHON_VERSION=3.10
ARG MLIO_VERSION=v0.9.0
ARG PYARROW_VERSION=17.0.0

ENV DEBIAN_FRONTEND=noninteractive

# Install uv and Python 3.10
RUN apt-get update && \
apt-get -y install --no-install-recommends \
build-essential curl git wget ca-certificates lsb-release software-properties-common && \
# Install uv
curl -LsSf https://astral.sh/uv/install.sh | sh && \
mv /root/.local/bin/uv /usr/local/bin/uv && \
# Install Python 3.10 with uv
uv python install 3.10 && \
ln -sf $(uv python find 3.10) /usr/bin/python3.10 && \
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1

# Add Apache Arrow repository (hardcoded for Ubuntu 20.04 focal)
RUN wget https://packages.apache.org/artifactory/arrow/ubuntu/apache-arrow-apt-source-latest-focal.deb && \
apt install -y -V ./apache-arrow-apt-source-latest-focal.deb && \
apt-get update && \
apt-get install -y -V libarrow-dev=17.0.0-1 libarrow-dataset-dev=17.0.0-1 libparquet-dev=17.0.0-1 libarrow-acero-dev=17.0.0-1 && \
# MLIO build dependencies
wget http://es.archive.ubuntu.com/ubuntu/pool/main/libf/libffi/libffi7_3.3-4_amd64.deb && \
dpkg -i libffi7_3.3-4_amd64.deb && \
apt-get -y install --no-install-recommends \
apt-transport-https gnupg && \
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \
gpg --dearmor - | \
tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null && \
echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ bionic main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null && \
apt-get update && \
rm /usr/share/keyrings/kitware-archive-keyring.gpg && \
apt-get install -y --no-install-recommends \
autoconf automake cmake cmake-data doxygen kitware-archive-keyring libcurl4-openssl-dev libssl-dev libtool ninja-build zlib1g-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

RUN ln -fs /usr/share/zoneinfo/UTC /etc/localtime && \
dpkg-reconfigure --frontend noninteractive tzdata

ENV PIP_ROOT_USER_ACTION=ignore

# Build MLIO from scratch
# Clone MLIO repository
RUN cd /tmp && \
git clone --branch ${MLIO_VERSION} https://github.com/awslabs/ml-io.git mlio

# Patch MLIO for Arrow 17.0.0
RUN cd /tmp/mlio && \
sed -i 's/find_package(Arrow 14.0.1 REQUIRED/find_package(Arrow 17.0.0 REQUIRED/g' CMakeLists.txt && \
sed -i 's/pyarrow==14.0.1/pyarrow==17.0.0/g' src/mlio-py/setup.py

# Build MLIO third-party dependencies (includes Arrow C++)
RUN cd /tmp/mlio && \
build-tools/build-dependency build/third-party all

# Configure MLIO build
RUN cd /tmp/mlio && \
mkdir -p build/release && \
cd build/release && \
cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_PREFIX_PATH="$(pwd)/../third-party" ../..

# Build MLIO core
RUN cd /tmp/mlio/build/release && \
cmake --build . && \
cmake --build . --target install

# Configure MLIO Python extension
RUN cd /tmp/mlio/build/release && \
cmake -DMLIO_INCLUDE_PYTHON_EXTENSION=ON -DPYTHON_EXECUTABLE="/usr/bin/python3" \
-DMLIO_INCLUDE_ARROW_INTEGRATION=ON -DCMAKE_PREFIX_PATH="$(pwd)/../third-party" ../..

# Build MLIO Python extension
RUN cd /tmp/mlio/build/release && \
cmake --build . --target mlio-py && \
cmake --build . --target mlio-arrow

# Build MLIO Python wheel
RUN cd /tmp/mlio/src/mlio-py && \
uv build

# Copy TBB libraries and MLIO shared libraries to a location we can copy from
RUN mkdir -p /mlio-artifacts && \
cp -r /tmp/mlio/build/third-party/lib/libtbb* /mlio-artifacts/ && \
cp /usr/local/lib/libmlio* /mlio-artifacts/ 2>/dev/null || true && \
cp /tmp/mlio/src/mlio-py/dist/*.whl /mlio-artifacts/

# Main image
FROM ubuntu:${UBUNTU_VERSION}@sha256:${UBUNTU_IMAGE_DIGEST}

ARG PYTHON_VERSION=3.10
ARG PYARROW_VERSION=17.0.0
ARG PYTHON_VERSION=3.11

ENV DEBIAN_FRONTEND=noninteractive

Expand All @@ -130,12 +37,12 @@ RUN apt-get update && \
apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
apt-get update && \
apt-get install -y -V libarrow-dev=17.0.0-1 libarrow-dataset-dev=17.0.0-1 libparquet-dev=17.0.0-1 libarrow-acero-dev=17.0.0-1 && \
# Add deadsnakes PPA for Python 3.10
# Add deadsnakes PPA for Python 3.11
add-apt-repository ppa:deadsnakes/ppa && \
apt-get update && \
apt-get -y install --no-install-recommends \
python3.10 python3.10-distutils python3.10-dev && \
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 && \
python3.11 python3.11-distutils python3.11-dev && \
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
curl -sS https://bootstrap.pypa.io/get-pip.py | python3 && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
Expand All @@ -150,15 +57,6 @@ RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \
ENV PATH=/usr/local/bin:${PATH}
ENV PIP_ROOT_USER_ACTION=ignore

# Copy MLIO wheel, TBB libraries, and MLIO shared libraries from builder stage
COPY --from=mlio-builder /mlio-artifacts/*.whl /tmp/
COPY --from=mlio-builder /mlio-artifacts/libtbb* /usr/local/lib/
COPY --from=mlio-builder /mlio-artifacts/libmlio* /usr/local/lib/

# Install MLIO wheel
RUN uv pip install --system /tmp/*.whl && \
rm /tmp/*.whl

# Copy compiled SQLite from builder stage
COPY --from=sqlite-builder /usr/local/bin/sqlite3 /usr/local/bin/sqlite3
COPY --from=sqlite-builder /usr/local/lib/libsqlite3.* /usr/local/lib/
Expand Down
6 changes: 3 additions & 3 deletions docker/1.4-2/final/Dockerfile.cpu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ COPY requirements.txt /requirements.txt
RUN uv pip install --system -r /requirements.txt && \
rm /requirements.txt

# Fix Python 3.10 compatibility for sagemaker-containers
RUN python3 -c "import sys; import os; site_packages = '/usr/local/lib/python3.10/dist-packages'; mapping_file = os.path.join(site_packages, 'sagemaker_containers/_mapping.py'); exec('if os.path.exists(mapping_file):\\n with open(mapping_file, \"r\") as f:\\n content = f.read()\\n content = content.replace(\"collections.Mapping\", \"collections.abc.Mapping\")\\n with open(mapping_file, \"w\") as f:\\n f.write(content)')"
# Fix Python 3.11 compatibility for sagemaker-containers
RUN python3 -c "import sys; import os; site_packages = '/usr/local/lib/python3.11/dist-packages'; mapping_file = os.path.join(site_packages, 'sagemaker_containers/_mapping.py'); exec('if os.path.exists(mapping_file):\\n with open(mapping_file, \"r\") as f:\\n content = f.read()\\n content = content.replace(\"collections.Mapping\", \"collections.abc.Mapping\")\\n with open(mapping_file, \"w\") as f:\\n f.write(content)')"

COPY dist/sagemaker_sklearn_container-2.0-py3-none-any.whl /sagemaker_sklearn_container-2.0-py3-none-any.whl
RUN uv pip install --system --no-cache /sagemaker_sklearn_container-2.0-py3-none-any.whl && \
Expand Down Expand Up @@ -56,4 +56,4 @@ EXPOSE 8080
ENV TEMP=/home/model-server/tmp

# Required label for multi-model loading
LABEL com.amazonaws.sagemaker.capabilities.multi-models=true
LABEL com.amazonaws.sagemaker.capabilities.multi-models=true
8 changes: 4 additions & 4 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@
name = "sagemaker-sklearn-container"
version = "2.0"
description = "SageMaker Scikit-learn Container"
requires-python = "==3.10.*"
license = "Apache-2.0"
requires-python = "==3.11.*"
license = {text = "Apache-2.0"}
authors = [{name = "Amazon Web Services"}]
classifiers = [
"Development Status :: 5 - Production/Stable",
"Intended Audience :: Developers",
"Natural Language :: English",
"Programming Language :: Python",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
]
dynamic = ["readme", "dependencies", "optional-dependencies"]

Expand All @@ -31,4 +31,4 @@ where = ["src"]
exclude = ["test*"]

[tool.setuptools.package-dir]
"" = "src"
"" = "src"
4 changes: 1 addition & 3 deletions tox.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,11 @@ deps =
-r{toxinidir}/test-requirements.txt
conda_deps=
pyarrow=14.0.1
mlio-py=0.9
conda_channels=
conda-forge
mlio
commands =
pytest --cov=sagemaker_sklearn_container --cov-fail-under=60 test/unit

[testenv:flake8]
deps = flake8
commands = flake8 setup.py src test
commands = flake8 setup.py src test