I have this Dockerfile:
# Base Image
FROM python:3.8-slim
LABEL maintainer="gizelly"
# Arguments that can be set with docker build
ARG AIRFLOW_VERSION=1.10.12
ARG AIRFLOW_HOME=/usr/local/airflow
# Export the environment variable AIRFLOW_HOME where airflow will be installed
ENV AIRFLOW_HOME=${AIRFLOW_HOME}
# Install dependencies and tools
RUN apt-get update -yqq && \
apt-get upgrade -yqq && \
apt-get install -yqq --no-install-recommends \
wget \
libczmq-dev \
curl \
libssl-dev \
git \
inetutils-telnet \
bind9utils freetds-dev \
libkrb5-dev \
libsasl2-dev \
libffi-dev libpq-dev \
freetds-bin build-essential \
default-libmysqlclient-dev \
apt-utils \
rsync \
zip \
unzip \
gcc \
vim \
locales \
&& apt-get clean
RUN apt-get update -y
RUN pip install deployv==0.9.173
RUN pip install psutil==5.7.3
RUN pip install python-dev-tools==2020.9.10
RUN apt-get update && apt-get install -y --no-install-recommends \
unixodbc-dev \
unixodbc \
libpq-dev
RUN python3.8 -m pip install pyodbc==4.0.30
COPY ./requirements-python3.8.txt /requirements-python3.8.txt
# Upgrade pip
# Create airflow user
# Install apache airflow with subpackages
RUN pip install --upgrade pip && \
useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow && \
pip install apache-airflow[microsoft.azure,azure_blob_storage,azure_cosmos,azure_data_lake,all_dbs,crypto,celery,postgres,kubernetes,docker]==${AIRFLOW_VERSION} --constraint /requirements-python3.8.txt
RUN python3.8 -m pip install azure-storage-blob --upgrade --force-reinstall
RUN python3.8 -m pip install azure-storage-blob --user
RUN python3.8 -m pip install azure-storage-blob --upgrade --force-reinstall
RUN python3.8 -c "from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient"
# Copy the entrypoint.sh from host to container (at path AIRFLOW_HOME)
COPY ./entrypoint.sh ./entrypoint.sh
# Set the entrypoint.sh file to be executable
RUN chmod +x ./entrypoint.sh
# Set the owner of the files in AIRFLOW_HOME to the user airflow
RUN chown -R airflow: ${AIRFLOW_HOME}
# Set the username to use
USER airflow
# Set workdir (it's like a cd inside the container)
WORKDIR ${AIRFLOW_HOME}
# Create the dags folder which will contain the DAGs
RUN mkdir dags
# Expose ports (just to indicate that this container needs to map port)
EXPOSE 8080
# Execute the entrypoint.sh
ENTRYPOINT [ "/entrypoint.sh" ]
And i have this requirements:
# Editable install with no version control (apache-airflow==1.10.12)
Babel==2.8.0
Flask-Admin==1.5.4
Flask-AppBuilder==2.3.4
Flask-Babel==1.0.0
Flask-Bcrypt==0.7.1
Flask-Caching==1.3.3
Flask-JWT-Extended==3.24.1
Flask-Login==0.4.1
Flask-OpenID==1.2.5
Flask-SQLAlchemy==2.4.4
Flask-WTF==0.14.3
Flask==1.1.2
JPype1==0.7.1
JayDeBeApi==1.2.3
Jinja2==2.11.2
Mako==1.1.3
Markdown==2.6.11
MarkupSafe==1.1.1
PyHive==0.6.3
PyJWT==1.7.1
PyNaCl==1.4.0
PySmbClient==0.1.5
PyYAML==5.3.1
Pygments==2.6.1
SQLAlchemy-JSONField==0.9.0
SQLAlchemy-Utils==0.36.8
SQLAlchemy==1.3.18
Sphinx==3.2.1
Unidecode==1.1.1
WTForms==2.3.3
Werkzeug==0.16.1
adal==1.2.4
alabaster==0.7.12
alembic==1.4.2
amqp==2.6.1
analytics-python==1.2.9
ansiwrap==0.8.4
apipkg==1.5
apispec==1.3.3
appdirs==1.4.4
argcomplete==1.12.0
asn1crypto==1.4.0
astroid==2.4.2
async-generator==1.10
atlasclient==1.0.0
attrs==19.3.0
aws-sam-translator==1.26.0
aws-xray-sdk==2.6.0
azure-common==1.1.25
azure-cosmos==3.2.0
azure-datalake-store==0.0.49
azure-mgmt-containerinstance==1.5.0
azure-mgmt-resource==10.2.0
azure-nspkg==3.0.2
azure-storage-blob==2.1.0
azure-storage-common==2.1.0
azure-storage==0.36.0
backcall==0.2.0
bcrypt==3.2.0
beautifulsoup4==4.7.1
billiard==3.6.3.0
black==19.10b0
blinker==1.4
boto3==1.14.43
boto==2.49.0
botocore==1.17.43
cached-property==1.5.1
cachetools==4.1.1
cassandra-driver==3.20.2
cattrs==1.0.0
celery==4.4.7
certifi==2020.6.20
cffi==1.14.2
cfgv==3.2.0
cfn-lint==0.35.0
cgroupspy==0.1.6
chardet==3.0.4
click==6.7
cloudant==0.5.10
colorama==0.4.3
colorlog==4.0.2
configparser==3.5.3
coverage==5.2.1
croniter==0.3.34
cryptography==3.0
cx-Oracle==8.0.0
datadog==0.38.0
decorator==4.4.2
defusedxml==0.6.0
dill==0.3.2
distlib==0.3.1
dnspython==1.16.0
docker-pycreds==0.4.0
docker==3.7.3
docopt==0.6.2
docutils==0.16
ecdsa==0.14.1
elasticsearch-dsl==5.4.0
elasticsearch==5.5.3
email-validator==1.1.1
entrypoints==0.3
execnet==1.7.1
fastavro==0.24.1
filelock==3.0.12
flake8-colors==0.1.6
flake8==3.8.3
flaky==3.7.0
flask-swagger==0.2.14
flower==0.9.5
freezegun==0.3.15
fsspec==0.8.0
funcsigs==1.0.2
future-fstrings==1.2.0
future==0.18.2
gcsfs==0.6.2
google-api-core==1.22.1
google-api-python-client==1.10.0
google-auth-httplib2==0.0.4
google-auth-oauthlib==0.4.1
google-auth==1.20.1
google-cloud-bigquery==1.26.1
google-cloud-bigtable==1.4.0
google-cloud-container==1.0.1
google-cloud-core==1.4.1
google-cloud-dlp==1.0.0
google-cloud-language==1.3.0
google-cloud-secret-manager==1.0.0
google-cloud-spanner==1.17.1
google-cloud-speech==1.3.2
google-cloud-storage==1.30.0
google-cloud-texttospeech==1.0.1
google-cloud-translate==1.7.0
google-cloud-videointelligence==1.15.0
google-cloud-vision==1.0.0
google-crc32c==0.1.0
google-resumable-media==0.7.1
googleapis-common-protos==1.52.0
graphviz==0.14.1
grpc-google-iam-v1==0.12.3
grpcio-gcp==0.2.2
grpcio==1.31.0
gunicorn==20.0.4
hdfs==2.5.8
hmsclient==0.1.1
httplib2==0.18.1
humanize==2.6.0
hvac==0.10.5
identify==1.4.28
idna==2.10
imagesize==1.2.0
importlib-metadata==1.7.0
inflection==0.5.0
ipdb==0.13.3
ipython-genutils==0.2.0
ipython==7.17.0
iso8601==0.1.12
isodate==0.6.0
itsdangerous==1.1.0
jedi==0.17.2
jira==2.0.0
jmespath==0.10.0
json-merge-patch==0.2
jsondiff==1.1.2
jsonpatch==1.26
jsonpickle==1.4.1
jsonpointer==2.0
jsonschema==3.2.0
junit-xml==1.9
jupyter-client==6.1.6
jupyter-core==4.6.3
kombu==4.6.11
kubernetes==11.0.0
lazy-object-proxy==1.5.1
ldap3==2.8
lockfile==0.12.2
marshmallow-enum==1.5.1
marshmallow-sqlalchemy==0.23.1
marshmallow==2.21.0
mccabe==0.6.1
mock==4.0.2
mongomock==3.20.0
more-itertools==8.4.0
moto==1.3.14
msrest==0.6.18
msrestazure==0.6.4
multi-key-dict==2.0.3
mypy-extensions==0.4.3
mypy==0.720
mysqlclient==1.3.14
natsort==7.0.1
nbclient==0.4.1
nbformat==5.0.7
nest-asyncio==1.4.0
networkx==2.4
nodeenv==1.4.0
nteract-scrapbook==0.4.1
ntlm-auth==1.5.0
numpy==1.19.1
oauthlib==3.1.0
oscrypto==1.2.1
packaging==20.4
pandas-gbq==0.13.2
pandas==1.1.0
papermill==2.1.2
parameterized==0.7.4
paramiko==2.7.1
parso==0.7.1
pathspec==0.8.0
pbr==5.4.5
pendulum==1.4.4
pexpect==4.8.0
pickleshare==0.7.5
pinotdb==0.1.1
pluggy==0.13.1
pre-commit==2.6.0
presto-python-client==0.7.0
prison==0.1.3
prometheus-client==0.8.0
prompt-toolkit==3.0.6
protobuf==3.13.0
psutil==5.7.2
psycopg2-binary==2.8.5
ptyprocess==0.6.0
py==1.9.0
pyOpenSSL==19.1.0
pyarrow==0.17.1
pyasn1-modules==0.2.8
pyasn1==0.4.8
pycodestyle==2.6.0
pycparser==2.20
pycryptodomex==3.9.8
pydata-google-auth==1.1.0
pydruid==0.5.8
pyflakes==2.2.0
pykerberos==1.2.1
pymongo==3.10.1
pyparsing==2.4.7
pyrsistent==0.16.0
pysftp==0.2.9
pytest-cov==2.10.1
pytest-forked==1.3.0
pytest-instafail==0.4.2
pytest-rerunfailures==9.0
pytest-timeout==1.4.2
pytest-xdist==2.0.0
pytest==5.4.3
python-daemon==2.2.4
python-dateutil==2.8.1
python-editor==1.0.4
python-http-client==3.2.7
python-jenkins==1.7.0
python-jose==3.2.0
python-nvd3==0.15.0
python-slugify==4.0.1
python3-openid==3.2.0
pytz==2020.1
pytzdata==2020.1
pywinrm==0.4.1
pyzmq==19.0.2
qds-sdk==1.16.0
redis==3.5.3
regex==2020.7.14
requests-futures==0.9.4
requests-kerberos==0.12.0
requests-mock==1.8.0
requests-ntlm==1.1.0
requests-oauthlib==1.3.0
requests-toolbelt==0.9.1
requests==2.24.0
responses==0.10.16
rsa==4.6
s3transfer==0.3.3
sasl==0.2.1
sendgrid==5.6.0
sentinels==1.0.0
sentry-sdk==0.16.5
setproctitle==1.1.10
six==1.15.0
slackclient==1.3.2
snowballstemmer==2.0.0
snowflake-connector-python==2.2.10
snowflake-sqlalchemy==1.2.3
soupsieve==2.0.1
sphinx-argparse==0.2.5
sphinx-autoapi==1.0.0
sphinx-copybutton==0.3.0
sphinx-jinja==1.1.1
sphinx-rtd-theme==0.5.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-dotnetdomain==0.4
sphinxcontrib-golangdomain==0.2.0.dev0
sphinxcontrib-htmlhelp==1.0.3
sphinxcontrib-httpdomain==1.7.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.4
sshpubkeys==3.1.0
sshtunnel==0.1.5
tabulate==0.8.7
tenacity==4.12.0
text-unidecode==1.3
textwrap3==0.9.2
thrift-sasl==0.4.2
thrift==0.13.0
toml==0.10.1
tornado==5.1.1
tqdm==4.48.2
traitlets==4.3.3
typed-ast==1.4.1
typing-extensions==3.7.4.2
tzlocal==1.5.1
unicodecsv==0.14.1
uritemplate==3.0.1
urllib3==1.25.10
vertica-python==0.11.0
vine==1.3.0
virtualenv==20.0.30
wcwidth==0.2.5
websocket-client==0.57.0
wrapt==1.12.1
xmltodict==0.12.0
yamllint==1.24.2
zdesk==2.7.1
zipp==3.1.0
zope.deprecation==4.4.0
pyodbc==4.0.30
My DAG:
from airflow import DAG
from datetime import datetime, timedelta
import airflow
from airflow.utils.dates import days_ago
from airflow.operators.bash_operator import BashOperator
from airflow.operators.dummy_operator import DummyOperator
# Argumentos básicos de uma DAG
default_args = {
'start_date':datetime(2020,12,3),
'owner':'precredito',
'depends_on_past': False,
'email': ['alexandre.brito@luminiitsolutions.com'],
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(seconds=5)
}
###### DAG #########
with DAG(dag_id='etl-motor-credito-verdadeiro-vitorioso.',
default_args=default_args,
schedule_interval=timedelta(minutes=10)) as dag:
start = DummyOperator(task_id='start')
etlmotor = BashOperator(task_id='blob',
bash_command="python3.8 /usr/local/airflow/dags/blob.py")
insertbd = BashOperator(task_id='azure',
bash_command="python3.8 /usr/local/airflow/dags/azure.py")
end = DummyOperator(task_id='end')
start >> etlmotor >> insertbd >> end
My script blob.py:
import pyodbc
import sys
sys.path.append(' /usr/local/lib/python3.8/site-packages')
import os, uuid
from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
import pandas as pd
print("test")
My script azure.py:
import pyodbc
import sys
sys.path.append(' /usr/local/lib/python3.8/site-packages')
import os, uuid
from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
import pandas as pd
print("test2")
I can raise the airflow normally, but when I run a script using a Bash command with bashoperator, the error below occurs:
File "/usr/local/airflow/dags/blob.py", line 5, in <module>
[2020-12-06 23:42:22,977] {bash_operator.py:157} INFO - from azure.storage.blob import BlobServiceClient
[2020-12-06 23:42:22,977] {bash_operator.py:157} INFO - File "/usr/local/airflow/dags/azure.py", line 5, in <module>
[2020-12-06 23:42:22,978] {bash_operator.py:157} INFO - from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
[2020-12-06 23:42:22,978] {bash_operator.py:157} INFO - ModuleNotFoundError: No module named 'azure.storage'; 'azure' is not a package
[2020-12-06 23:42:22,991] {bash_operator.py:159} INFO - Command exited with return code 1
[2020-12-06 23:42:23,002] {taskinstance.py:1150} ERROR - Bash command failed
Traceback (most recent call last):
It appears that the container does not recognize the installed package and is unable to import. How can I solve this? Thank you!