Use python 3.8.6
Install python 3.8.6
$ pyenv install 3.8.6
Get poetry
$ curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python
$ mkdir <path-to-poetry-project>
$ cd <path-to-poetry-project>
Set python versions in the directory
$ pyenv local 3.8.6
$ poetry init
Modify pyproject.toml
(Install the major machine learning libraries)
[tool.poetry]
name = "your-project-name"
version = "0.1.0"
description = ""
authors = ["Your Name <you@example.com>"]
[tool.poetry.dependencies]
python = "3.8.6"
torch = "1.9.1"
transformers = "4.12.5"
sklearn = "^0.0"
pandas = "^1.3.5"
matplotlib = "^3.5.1"
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
Create virtual env in project directory(in default .cache)
$ poetry config virtualenvs.in-project true --local
Install python package
$ poetry install
example to run python code in compute nodes
#$ -S /usr/bin/bash
#$ -cwd
#$ -pe x64 64
#$ -j y
#$ -N python
export PYENV_ROOT="$HOME/.pyenv"
export PATH="$PYENV_ROOT/bin:$PATH"
eval "$(pyenv init -)"
export PATH="/home/ueda/.poetry/bin:$PATH"
export PYTHONPATH="<path-to-your-poetry-project>/.venv/lib/python3.7/site-packages:${PYTHONPATH}"
source /opt/settings/2017.4/intel-compiler.sh
source /opt/settings/2017.4/intel-mpi.sh
export I_MPI_PIN=1
poetry run python <pyhton-file>