这是我想出的解决方案：
文件夹结构：
.
├── docker-compose.yaml
├── airflow
│ └── Dockerfile
├── dags
│ └── papermill_test.py
└── papermill
├── Dockerfile
├── environments
│ ├── requirements.txt
│ └── environments.yml
├── notebooks
│ └── papermill.test.ipynb
└── output
docker-compose.yaml
version: '3'
x-airflow-common:
&airflow-common
build:
context: ./airflow
args:
- DOCKER_UID=${DOCKER_UID-1000} # build args: DOCKER_UID=`id -u`
environment:
&airflow-common-env
AIRFLOW__CORE__EXECUTOR: CeleryExecutor
AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow
AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
AIRFLOW__CORE__FERNET_KEY: ''
AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth'
_PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
volumes:
- /var/run/docker.sock:/var/run/docker.sock:rw # run this on localhost: sudo chmod 666 /var/run/docker.sock
- ./dags:/opt/airflow/dags
- ./logs:/opt/airflow/logs
- ./plugins:/opt/airflow/plugins
- ./papermill/notebooks/:/opt/airflow/notebooks/
- ./papermill/output/:/opt/airflow/output/
user: "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-50000}"
depends_on:
redis:
condition: service_healthy
postgres:
condition: service_healthy
services:
postgres:
image: postgres:12
environment:
POSTGRES_USER: airflow
POSTGRES_PASSWORD: airflow
POSTGRES_DB: airflow
volumes:
- postgres-db-volume:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "pg_isready", "-U", "airflow"]
interval: 5s
retries: 5
restart: always
redis:
image: redis:latest
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 30s
retries: 50
restart: always
airflow-webserver:
<<: *airflow-common
command: webserver
ports:
- 8080:8080
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
interval: 10s
timeout: 10s
retries: 5
restart: always
airflow-scheduler:
<<: *airflow-common
command: scheduler
healthcheck:
test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"']
interval: 10s
timeout: 10s
retries: 5
restart: always
airflow-worker:
<<: *airflow-common
command: celery worker
healthcheck:
test:
- "CMD-SHELL"
- 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
interval: 10s
timeout: 10s
retries: 5
restart: always
airflow-init:
<<: *airflow-common
command: version
environment:
<<: *airflow-common-env
_AIRFLOW_DB_UPGRADE: 'true'
_AIRFLOW_WWW_USER_CREATE: 'true'
_AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
_AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
flower:
<<: *airflow-common
command: celery flower
ports:
- 5555:5555
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
interval: 10s
timeout: 10s
retries: 5
restart: always
papermill:
build: ./papermill
image: your_name/papermill
volumes:
- ./papermill/notebooks/:/opt/airflow/notebooks
- ./papermill/output/:/opt/airflow/output/
volumes:
postgres-db-volume:
气流/Dockerfile
FROM apache/airflow:2.1.3
USER root
# This fixes permission issues on linux.
# The airflow user should have the same UID as the user running docker on the host system.
# make build is adjust this value automatically
ARG DOCKER_UID
RUN \
: "${DOCKER_UID:?Build argument DOCKER_UID needs to be set and non-empty. Use 'make build' to set it automatically.}" \
&& usermod -u ${DOCKER_UID} airflow \
&& find / -path /proc -prune -o -user 50000 -exec chown -h airflow {} \; \
&& echo "Set airflow's uid to ${DOCKER_UID}"
RUN sudo addgroup --system docker
RUN sudo adduser airflow docker
RUN newgrp docker
USER airflow
RUN pip install apache-airflow-providers-docker
dags/papermill_test.py
from datetime import timedelta
from airflow import DAG
from airflow.providers.docker.operators.docker import DockerOperator
from airflow.utils.dates import days_ago
from docker.types import Mount
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'email': ['your@email.com'],
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
with DAG('papermill_test',
default_args=default_args,
schedule_interval=timedelta(days=1),
start_date=days_ago(2),
catchup=False,
) as dag:
run_this = DockerOperator(
image='your_name/papermill:latest',
task_id='docker_command',
api_version='auto',
auto_remove=True,
command="papermill /opt/airflow/notebooks/papermill.test.ipynb /opt/airflow/output/papermill.test.ipynb -k papermill -p a 2 -p twice 5",
docker_url='unix://var/run/docker.sock',
network_mode='airflow_default',
mount_tmp_dir=False,
mounts=[Mount(target='/opt/airflow/notebooks/',
source='/<project_folder>/airflow/papermill/notebooks/', # todo change it when in prod
type="bind"),
Mount(target='/opt/airflow/data/',
source='/<project_folder>/airflow/papermill/data/', # todo change it when in prod
type="bind"),
Mount(target='/opt/airflow/output/',
source='/<project_folder>/airflow/papermill/output/', # todo change it when in prod
type="bind"), ]
)
造纸厂/Dockerfile
FROM continuumio/miniconda3:4.10.3-alpine
WORKDIR /opt/airflow/
# install environment
COPY ./environments/environment.yml environment.yml
COPY ./environments/requirements.txt requirements.txt
RUN conda env create -f environment.yml
RUN . /root/.bashrc && \
conda init bash && \
conda activate papermill && \
pip install -r requirements.txt && \
python -m ipykernel install --user --name=papermill
RUN pip install ipykernel papermill
COPY ./notebooks /opt/airflow/notebooks
COPY ./output /opt/airflow/output
COPY ./data /opt/airflow/data
CMD ["papermill", "--help-notebook", "/opt/airflow/notebooks/papermill.test.ipynb"]
papermill/environments/requirements.txt
您可以使用通常的 pip install -r requirements.txt 文件。
papermill/environments/environments.yml
name: papermill
channels:
- conda-forge
- defaults
dependencies:
- tqdm
- psycopg2
- ipykernel
- papermill
- ipywidgets
prefix: /opt/airflow/envs/papermill
papermill/notebooks/papermill.test.ipynb
这里可以使用示例here中的代码。
用法：
您可以使用 papermill_test.py 文件中的代码作为模板来创建新的 dag。为了在容器中执行 papermill 记得更改命令参数如下：
command="papermill /opt/airflow/notebooks/name_of_your_notebook.ipynb /opt/airflow/output/output_notebook.ipynb -k papermill -p param1 1 -p param2 -p param3 3",
不要忘记使用以下命令从 docker 容器外部更改 /var/run/docker.sock 文件的权限：
sudo chmod 666 /var/run/docker.sock
不幸的是，我还没有找到任何其他合适的解决方案。我愿意接受建议。