pytorch / pytorch Public

zhouzhuojie Add --jobs 0 for git submodule update (#61311 )

Latest commit 6107cf3

Jul 7, 2021

Summary:
Pull Request resolved: #61311

Pull Request resolved: #61152

Some related docs about `submodule.fetchJobs`
https://git-scm.com/docs/git-config#Documentation/git-config.txt-submodulefetchJobs

```
time git submodule update --init --recursive
________________________________________________________
Executed in  243.20 secs    fish           external
   usr time   49.64 secs  213.00 micros   49.64 secs
   sys time   29.27 secs  795.00 micros   29.27 secs
```

```
time git submodule update --init --recursive --jobs 4
________________________________________________________
Executed in  143.04 secs    fish           external
   usr time   51.06 secs  246.00 micros   51.06 secs
   sys time   30.96 secs  742.00 micros   30.96 secs
```

```
time git submodule update --init --recursive --jobs 8
________________________________________________________
Executed in  124.64 secs    fish           external
   usr time   51.76 secs  264.00 micros   51.76 secs
   sys time   30.49 secs  739.00 micros   30.49 secs

```

```
time git submodule update --init --recursive --jobs 0 # use all online cpus
 ________________________________________________________
Executed in  129.75 secs    fish           external
   usr time   51.64 secs  181.00 micros   51.64 secs
   sys time   31.49 secs  781.00 micros   31.49 secs

```

Test Plan: Imported from OSS

Reviewed By: 1ntEgr8

Differential Revision: D29560875

Pulled By: zhouzhuojie

fbshipit-source-id: 556027dffe744c66428075a8a1bf64683930aaaf

11 contributors

Users who have contributed to this file

81 lines (74 sloc) 2.86 KB

Raw Blame

	# syntax = docker/dockerfile:experimental
	#
	# NOTE: To build this you will need a docker version > 18.06 with
	# experimental enabled and DOCKER_BUILDKIT=1
	#
	# If you do not use buildkit you are not going to have a good time
	#
	# For reference:
	# https://docs.docker.com/develop/develop-images/build_enhancements/
	ARG BASE_IMAGE=ubuntu:18.04
	ARG PYTHON_VERSION=3.8

	FROM ${BASE_IMAGE} as dev-base
	RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \
	apt-get update && apt-get install -y --no-install-recommends \
	build-essential \
	ca-certificates \
	ccache \
	cmake \
	curl \
	git \
	libjpeg-dev \
	libpng-dev && \
	rm -rf /var/lib/apt/lists/*
	RUN /usr/sbin/update-ccache-symlinks
	RUN mkdir /opt/ccache && ccache --set-config=cache_dir=/opt/ccache
	ENV PATH /opt/conda/bin:$PATH

	FROM dev-base as conda
	ARG PYTHON_VERSION=3.8
	RUN curl -fsSL -v -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
	chmod +x ~/miniconda.sh && \
	~/miniconda.sh -b -p /opt/conda && \
	rm ~/miniconda.sh && \
	/opt/conda/bin/conda install -y python=${PYTHON_VERSION} conda-build pyyaml numpy ipython&& \
	/opt/conda/bin/conda clean -ya

	FROM dev-base as submodule-update
	WORKDIR /opt/pytorch
	COPY . .
	RUN git submodule update --init --recursive --jobs 0

	FROM conda as build
	WORKDIR /opt/pytorch
	COPY --from=conda /opt/conda /opt/conda
	COPY --from=submodule-update /opt/pytorch /opt/pytorch
	RUN --mount=type=cache,target=/opt/ccache \
	TORCH_CUDA_ARCH_LIST="3.5 5.2 6.0 6.1 7.0+PTX 8.0" TORCH_NVCC_FLAGS="-Xfatbin -compress-all" \
	CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" \
	python setup.py install

	FROM conda as conda-installs
	ARG PYTHON_VERSION=3.8
	ARG CUDA_VERSION=11.1
	ARG CUDA_CHANNEL=nvidia
	ARG INSTALL_CHANNEL=pytorch-nightly
	ENV CONDA_OVERRIDE_CUDA=${CUDA_VERSION}
	RUN /opt/conda/bin/conda install -c "${INSTALL_CHANNEL}" -c "${CUDA_CHANNEL}" -y python=${PYTHON_VERSION} pytorch torchvision torchtext "cudatoolkit=${CUDA_VERSION}" && \
	/opt/conda/bin/conda clean -ya
	RUN /opt/conda/bin/pip install torchelastic

	FROM ${BASE_IMAGE} as official
	ARG PYTORCH_VERSION
	LABEL com.nvidia.volumes.needed="nvidia_driver"
	RUN --mount=type=cache,id=apt-final,target=/var/cache/apt \
	apt-get update && apt-get install -y --no-install-recommends \
	ca-certificates \
	libjpeg-dev \
	libpng-dev && \
	rm -rf /var/lib/apt/lists/*
	COPY --from=conda-installs /opt/conda /opt/conda
	ENV PATH /opt/conda/bin:$PATH
	ENV NVIDIA_VISIBLE_DEVICES all
	ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
	ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
	ENV PYTORCH_VERSION ${PYTORCH_VERSION}
	WORKDIR /workspace

	FROM official as dev
	# Should override the already installed version from the official-image stage
	COPY --from=build /opt/conda /opt/conda

Feb	OCT	Feb
	05
2020	2021	2022

pytorch / pytorch Public

pytorch/Dockerfile

Users who have contributed to this file