More CI cleanup, add style workflow (#107)

- Changes on the `test.yml` workflow:
  - Using poetry instead of pip. Contrary to what I wrote in #75, it is possible to use poetry (and have the benefits of shorter install times) without the need for having two separate versions of `pyproject.toml` and `poetry.lock`.
  - Reduce the trigger scope to only run when files in these directories are modified:
    - `lerobot/`
    - `tests/`
    - `examples/`
    - `.github/`
- Add `style.yml` workflow for doing a `ruff check` pass on the code
- More cleanup (removed deprecated workflow)
这个提交包含在:
Simon Alibert
2024-04-27 09:37:56 +02:00
提交者 GitHub
父节点 45f351c618
当前提交 fdf6a0c4e3
共有 12 个文件被更改,包括 85 次插入 和 4188 次删除

3917
.github/poetry/cpu_DEPRECATED/poetry.lock 自动生成的 vendored

文件差异内容过多而无法显示 加载差异

查看文件

@ -1,107 +0,0 @@
[tool.poetry]
name = "lerobot"
version = "0.1.0"
description = "🤗 LeRobot: State-of-the-art Machine Learning for Real-World Robotics in Pytorch"
authors = [
"Rémi Cadène <re.cadene@gmail.com>",
"Alexander Soare <alexander.soare159@gmail.com>",
"Quentin Gallouédec <quentin.gallouedec@ec-lyon.fr>",
"Simon Alibert <alibert.sim@gmail.com>",
"Thomas Wolf <thomaswolfcontact@gmail.com>",
]
repository = "https://github.com/huggingface/lerobot"
readme = "README.md"
license = "Apache-2.0"
classifiers=[
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"Intended Audience :: Education",
"Intended Audience :: Science/Research",
"Topic :: Software Development :: Build Tools",
"Topic :: Scientific/Engineering :: Artificial Intelligence",
"License :: OSI Approved :: Apache Software License",
"Programming Language :: Python :: 3.10",
]
packages = [{include = "lerobot"}]
[tool.poetry.dependencies]
python = "^3.10"
termcolor = "^2.4.0"
omegaconf = "^2.3.0"
wandb = "^0.16.3"
imageio = {extras = ["ffmpeg"], version = "^2.34.0"}
gdown = "^5.1.0"
hydra-core = "^1.3.2"
einops = "^0.7.0"
pymunk = "^6.6.0"
zarr = "^2.17.0"
numba = "^0.59.0"
torch = {version = "^2.2.1", source = "torch-cpu"}
opencv-python = "^4.9.0.80"
diffusers = "^0.26.3"
torchvision = {version = "^0.17.1", source = "torch-cpu"}
h5py = "^3.10.0"
huggingface-hub = "^0.21.4"
robomimic = "0.2.0"
gymnasium = "^0.29.1"
cmake = "^3.29.0.1"
gym-pusht = { git = "git@github.com:huggingface/gym-pusht.git", optional = true}
gym-xarm = { git = "git@github.com:huggingface/gym-xarm.git", optional = true}
gym-aloha = { git = "git@github.com:huggingface/gym-aloha.git", optional = true}
pre-commit = {version = "^3.7.0", optional = true}
debugpy = {version = "^1.8.1", optional = true}
pytest = {version = "^8.1.0", optional = true}
pytest-cov = {version = "^5.0.0", optional = true}
datasets = "^2.19.0"
[tool.poetry.extras]
pusht = ["gym-pusht"]
xarm = ["gym-xarm"]
aloha = ["gym-aloha"]
dev = ["pre-commit", "debugpy"]
test = ["pytest", "pytest-cov"]
[[tool.poetry.source]]
name = "torch-cpu"
url = "https://download.pytorch.org/whl/cpu"
priority = "supplemental"
[tool.ruff]
line-length = 110
target-version = "py310"
exclude = [
".bzr",
".direnv",
".eggs",
".git",
".git-rewrite",
".hg",
".mypy_cache",
".nox",
".pants.d",
".pytype",
".ruff_cache",
".svn",
".tox",
".venv",
"__pypackages__",
"_build",
"buck-out",
"build",
"dist",
"node_modules",
"venv",
]
[tool.ruff.lint]
select = ["E4", "E7", "E9", "F", "I", "N", "B", "C4", "SIM"]
[build-system]
requires = ["poetry-core>=1.5.0"]
build-backend = "poetry.core.masonry.api"

查看文件

@ -1,6 +1,6 @@
# Inspired by # Inspired by
# https://github.com/huggingface/peft/blob/main/.github/workflows/build_docker_images.yml # https://github.com/huggingface/peft/blob/main/.github/workflows/build_docker_images.yml
name: Nightly Builds name: Builds
on: on:
workflow_dispatch: workflow_dispatch:
@ -8,10 +8,6 @@ on:
schedule: schedule:
- cron: "0 1 * * *" - cron: "0 1 * * *"
# concurrency:
# group: docker-image-builds
# cancel-in-progress: false
env: env:
PYTHON_VERSION: "3.10" PYTHON_VERSION: "3.10"
# CI_SLACK_CHANNEL: ${{ secrets.CI_DOCKER_CHANNEL }} # CI_SLACK_CHANNEL: ${{ secrets.CI_DOCKER_CHANNEL }}
@ -40,6 +36,8 @@ jobs:
- name: Check out code - name: Check out code
uses: actions/checkout@v4 uses: actions/checkout@v4
# HACK(aliberts): to be removed for release
# -----------------------------------------
- name: Checkout gym-aloha - name: Checkout gym-aloha
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
@ -61,8 +59,6 @@ jobs:
path: envs/gym-pusht path: envs/gym-pusht
ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
# HACK(aliberts): to be removed for release
# -----------------------------------------
- name: Set up Python 3.10 - name: Set up Python 3.10
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
@ -134,6 +130,8 @@ jobs:
- name: Check out code - name: Check out code
uses: actions/checkout@v4 uses: actions/checkout@v4
# HACK(aliberts): to be removed for release
# -----------------------------------------
- name: Checkout gym-aloha - name: Checkout gym-aloha
uses: actions/checkout@v4 uses: actions/checkout@v4
with: with:
@ -155,8 +153,6 @@ jobs:
path: envs/gym-pusht path: envs/gym-pusht
ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
# HACK(aliberts): to be removed for release
# -----------------------------------------
- name: Set up Python 3.10 - name: Set up Python 3.10
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:

查看文件

@ -1,6 +1,6 @@
# Inspired by # Inspired by
# https://github.com/huggingface/peft/blob/main/.github/workflows/nightly.yml # https://github.com/huggingface/peft/blob/main/.github/workflows/nightly.yml
name: Nightly Tests name: Nightly
on: on:
workflow_dispatch: workflow_dispatch:
@ -11,7 +11,6 @@ env:
DATA_DIR: tests/data DATA_DIR: tests/data
# SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }} # SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }}
jobs: jobs:
run_all_tests_cpu: run_all_tests_cpu:
name: "Test CPU" name: "Test CPU"
@ -33,11 +32,13 @@ jobs:
env: env:
DATA_DIR: tests/data DATA_DIR: tests/data
run: pytest -v --cov=./lerobot --disable-warnings tests run: pytest -v --cov=./lerobot --disable-warnings tests
- name: Tests end-to-end - name: Tests end-to-end
env: env:
DATA_DIR: tests/data DATA_DIR: tests/data
run: make test-end-to-end run: make test-end-to-end
run_all_tests_single_gpu: run_all_tests_single_gpu:
name: "Test GPU" name: "Test GPU"
strategy: strategy:
@ -59,6 +60,7 @@ jobs:
steps: steps:
- name: Nvidia-smi - name: Nvidia-smi
run: nvidia-smi run: nvidia-smi
- name: Test - name: Test
run: pytest -v --cov=./lerobot --cov-report=xml --disable-warnings tests run: pytest -v --cov=./lerobot --cov-report=xml --disable-warnings tests
# TODO(aliberts): Link with HF Codecov account # TODO(aliberts): Link with HF Codecov account
@ -69,14 +71,6 @@ jobs:
# verbose: true # verbose: true
- name: Tests end-to-end - name: Tests end-to-end
run: make test-end-to-end run: make test-end-to-end
- name: Tailscale Wait
if: ${{ failure() || runner.debug == '1' }}
uses: huggingface/tailscale-action@v1
with:
waitForSSH: true
authkey: ${{ secrets.TAILSCALE_SSH_AUTHKEY }}
slackChannel: ${{ secrets.SLACK_CIFEEDBACK_CHANNEL }}
slackToken: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
# - name: Generate Report # - name: Generate Report
# if: always() # if: always()

38
.github/workflows/style.yml vendored 普通文件
查看文件

@ -0,0 +1,38 @@
name: Style
on:
workflow_dispatch:
workflow_call:
pull_request:
branches:
- main
push:
branches:
- main
env:
PYTHON_VERSION: "3.10"
jobs:
ruff_check:
runs-on: ubuntu-latest
steps:
- name: Checkout Repository
uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
- name: Get Ruff Version from pre-commit-config.yaml
id: get-ruff-version
run: |
RUFF_VERSION=$(awk '/repo: https:\/\/github.com\/astral-sh\/ruff-pre-commit/{flag=1;next}/rev:/{if(flag){print $2;exit}}' .pre-commit-config.yaml)
echo "RUFF_VERSION=${RUFF_VERSION}" >> $GITHUB_ENV
- name: Install Ruff
run: python -m pip install "ruff==${{ env.RUFF_VERSION }}"
- name: Run Ruff
run: ruff check .

查看文件

@ -1,120 +0,0 @@
# name: Tests poetry
# on:
# pull_request:
# branches:
# - main
# push:
# branches:
# - main
# jobs:
# tests:
# runs-on: ubuntu-latest
# env:
# POETRY_VERSION: 1.8.2
# DATA_DIR: tests/data
# MUJOCO_GL: egl
# steps:
# #----------------------------------------------
# # check-out repo and set-up python
# #----------------------------------------------
# - name: Check out repository
# uses: actions/checkout@v4
# with:
# lfs: true
# - name: Set up python
# id: setup-python
# uses: actions/setup-python@v5
# with:
# python-version: '3.10'
# - name: Add SSH key for installing envs
# uses: webfactory/ssh-agent@v0.9.0
# with:
# ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
# #----------------------------------------------
# # install & configure poetry
# #----------------------------------------------
# - name: Load cached Poetry installation
# id: restore-poetry-cache
# uses: actions/cache/restore@v3
# with:
# path: ~/.local
# key: poetry-${{ env.POETRY_VERSION }}
# - name: Install Poetry
# if: steps.restore-poetry-cache.outputs.cache-hit != 'true'
# uses: snok/install-poetry@v1
# with:
# version: ${{ env.POETRY_VERSION }}
# virtualenvs-create: true
# installer-parallel: true
# - name: Save cached Poetry installation
# if: |
# steps.restore-poetry-cache.outputs.cache-hit != 'true' &&
# github.ref_name == 'main'
# id: save-poetry-cache
# uses: actions/cache/save@v3
# with:
# path: ~/.local
# key: poetry-${{ env.POETRY_VERSION }}
# - name: Configure Poetry
# run: poetry config virtualenvs.in-project true
# #----------------------------------------------
# # install dependencies
# #----------------------------------------------
# # TODO(aliberts): move to gpu runners
# - name: Select cpu dependencies # HACK
# run: cp -t . .github/poetry/cpu/pyproject.toml .github/poetry/cpu/poetry.lock
# - name: Load cached venv
# id: restore-dependencies-cache
# uses: actions/cache/restore@v3
# with:
# path: .venv
# key: venv-${{ steps.setup-python.outputs.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}
# - name: Install dependencies
# if: steps.restore-dependencies-cache.outputs.cache-hit != 'true'
# run: poetry install --no-interaction --no-root --all-extras
# - name: Save cached venv
# if: |
# steps.restore-dependencies-cache.outputs.cache-hit != 'true' &&
# github.ref_name == 'main'
# id: save-dependencies-cache
# uses: actions/cache/save@v3
# with:
# path: .venv
# key: venv-${{ steps.setup-python.outputs.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }}
# - name: Install EGL
# run: sudo apt-get update && sudo apt-get install -y libegl1-mesa-dev
# #----------------------------------------------
# # install project
# #----------------------------------------------
# - name: Install project
# run: poetry install --no-interaction --all-extras
# #----------------------------------------------
# # run tests & coverage
# #----------------------------------------------
# - name: Run tests
# run: |
# source .venv/bin/activate
# pytest -v --cov=./lerobot tests
# #----------------------------------------------
# # run end-to-end tests
# #----------------------------------------------
# - name: Tests end-to-end
# run: |
# source .venv/bin/activate
# make test-end-to-end

查看文件

@ -4,9 +4,19 @@ on:
pull_request: pull_request:
branches: branches:
- main - main
paths:
- "lerobot/**"
- "tests/**"
- "examples/**"
- ".github/**"
push: push:
branches: branches:
- main - main
paths:
- "lerobot/**"
- "tests/**"
- "examples/**"
- ".github/**"
jobs: jobs:
tests: tests:
@ -21,26 +31,27 @@ jobs:
ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with:
lfs: true - name: Install EGL
run: sudo apt-get update && sudo apt-get install -y libegl1-mesa-dev
- name: Install poetry
run: |
pipx install poetry && poetry config virtualenvs.in-project true
echo "${{ github.workspace }}/.venv/bin" >> $GITHUB_PATH
- name: Set up Python 3.10 - name: Set up Python 3.10
uses: actions/setup-python@v5 uses: actions/setup-python@v5
with: with:
python-version: "3.10" python-version: "3.10"
cache: ${{ (github.ref == 'refs/heads/main') && 'pip' || '' }} cache: "poetry"
cache-dependency-path: pyproject.toml
- name: Install EGL - name: Install poetry dependencies
run: sudo apt-get update && sudo apt-get install -y libegl1-mesa-dev
- name: Install pip dependencies
run: | run: |
python -m pip install --upgrade pip poetry install --all-extras
pip install -e ".[test, aloha, xarm, pusht]"
- name: Test with pytest - name: Test with pytest
run: pytest -v --cov=./lerobot tests run: pytest -v --cov=./lerobot --durations=0 tests
- name: Test end-to-end - name: Test end-to-end
run: make test-end-to-end run: make test-end-to-end

查看文件

@ -1,4 +1,4 @@
exclude: ^(data/|tests/data) exclude: ^(tests/data)
default_language_version: default_language_version:
python: python3.10 python: python3.10
repos: repos:
@ -18,7 +18,7 @@ repos:
hooks: hooks:
- id: pyupgrade - id: pyupgrade
- repo: https://github.com/astral-sh/ruff-pre-commit - repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.3.7 rev: v0.4.2
hooks: hooks:
- id: ruff - id: ruff
args: [--fix] args: [--fix]

查看文件

@ -153,17 +153,6 @@ Follow these steps to start contributing:
poetry lock --no-update poetry lock --no-update
``` ```
**NOTE:** Currently, to ensure the CI works properly, any new package must also be added in the CPU-only environment dedicated to the CI. To do this, you should create a separate environment and add the new package there as well. For example:
```bash
# Add the new package to your main poetry env
poetry add some-package
# Add the same package to the CPU-only env dedicated to CI
conda create -y -n lerobot-ci python=3.10
conda activate lerobot-ci
cd .github/poetry/cpu
poetry add some-package
```
5. Develop the features on your branch. 5. Develop the features on your branch.
As you work on the features, you should make sure that the test suite As you work on the features, you should make sure that the test suite

查看文件

@ -1,5 +1,16 @@
.PHONY: tests .PHONY: tests
PYTHON_PATH := $(shell which python)
# If Poetry is installed, redefine PYTHON_PATH to use the Poetry-managed Python
POETRY_CHECK := $(shell command -v poetry)
ifneq ($(POETRY_CHECK),)
PYTHON_PATH := $(shell poetry run which python)
endif
export PATH := $(dir $(PYTHON_PATH)):$(PATH)
build-cpu: build-cpu:
docker build -t lerobot:latest -f docker/lerobot-cpu/Dockerfile . docker build -t lerobot:latest -f docker/lerobot-cpu/Dockerfile .

查看文件

@ -68,6 +68,7 @@ test = ["pytest", "pytest-cov"]
line-length = 110 line-length = 110
target-version = "py310" target-version = "py310"
exclude = [ exclude = [
"tests/data",
".bzr", ".bzr",
".direnv", ".direnv",
".eggs", ".eggs",

查看文件

@ -1,5 +1,6 @@
# TODO(aliberts): Mute logging for these tests # TODO(aliberts): Mute logging for these tests
import subprocess import subprocess
import sys
from pathlib import Path from pathlib import Path
@ -11,7 +12,7 @@ def _find_and_replace(text: str, finds_and_replaces: list[tuple[str, str]]) -> s
def _run_script(path): def _run_script(path):
subprocess.run(["python", path], check=True) subprocess.run([sys.executable, path], check=True)
def test_example_1(): def test_example_1():