From f0cbe4a7de296a584484d6980efbf85c92f7e70a Mon Sep 17 00:00:00 2001 From: Amritanshu Date: Sun, 6 Aug 2023 09:09:02 +0530 Subject: [PATCH] Chore: Started using gunicorn --- brewman/brewman/__main__.py | 2 +- brewman/gunicorn.conf.py | 68 +++++++++++++++++++++++++++++++++++++ brewman/logging.conf | 53 +++++++++++++++++++++++++++++ brewman/pyproject.toml | 13 +++---- brewman/run.sh | 3 ++ docker/app/Dockerfile | 2 +- 6 files changed, 133 insertions(+), 8 deletions(-) create mode 100644 brewman/gunicorn.conf.py create mode 100644 brewman/logging.conf create mode 100755 brewman/run.sh diff --git a/brewman/brewman/__main__.py b/brewman/brewman/__main__.py index 3466e769..0d0c320d 100644 --- a/brewman/brewman/__main__.py +++ b/brewman/brewman/__main__.py @@ -1,4 +1,4 @@ -from brewman.main import init +from .main import init init() diff --git a/brewman/gunicorn.conf.py b/brewman/gunicorn.conf.py new file mode 100644 index 00000000..45fc1fba --- /dev/null +++ b/brewman/gunicorn.conf.py @@ -0,0 +1,68 @@ +import json +import multiprocessing +import os + + +workers_per_core_str = os.getenv("WORKERS_PER_CORE", "1") +max_workers_str = os.getenv("MAX_WORKERS") +use_max_workers = None +if max_workers_str: + use_max_workers = int(max_workers_str) +web_concurrency_str = os.getenv("WEB_CONCURRENCY", None) + +host = os.getenv("HOST", "0.0.0.0") +port = os.getenv("PORT", "9994") +bind_env = os.getenv("BIND", None) +use_loglevel = os.getenv("LOG_LEVEL", "info") +if bind_env: + use_bind = bind_env +else: + use_bind = f"{host}:{port}" + +cores = multiprocessing.cpu_count() +workers_per_core = float(workers_per_core_str) +default_web_concurrency = workers_per_core * cores +if web_concurrency_str: + web_concurrency = int(web_concurrency_str) + assert web_concurrency > 0 +else: + web_concurrency = max(int(default_web_concurrency), 2) + if use_max_workers: + web_concurrency = min(web_concurrency, use_max_workers) +accesslog_var = os.getenv("ACCESS_LOG", "-") +use_accesslog = accesslog_var or None +errorlog_var = os.getenv("ERROR_LOG", "-") +use_errorlog = errorlog_var or None +graceful_timeout_str = os.getenv("GRACEFUL_TIMEOUT", "120") +timeout_str = os.getenv("TIMEOUT", "120") +keepalive_str = os.getenv("KEEP_ALIVE", "5") + +# Gunicorn config variables +loglevel = use_loglevel +workers = web_concurrency +bind = use_bind +errorlog = use_errorlog +worker_tmp_dir = "/dev/shm" +accesslog = use_accesslog +graceful_timeout = int(graceful_timeout_str) +timeout = int(timeout_str) +keepalive = int(keepalive_str) + + +# For debugging and testing +log_data = { + "loglevel": loglevel, + "workers": workers, + "bind": bind, + "graceful_timeout": graceful_timeout, + "timeout": timeout, + "keepalive": keepalive, + "errorlog": errorlog, + "accesslog": accesslog, + # Additional, non-gunicorn variables + "workers_per_core": workers_per_core, + "use_max_workers": use_max_workers, + "host": host, + "port": port, +} +print(json.dumps(log_data)) diff --git a/brewman/logging.conf b/brewman/logging.conf new file mode 100644 index 00000000..07828799 --- /dev/null +++ b/brewman/logging.conf @@ -0,0 +1,53 @@ +[loggers] +keys=root, gunicorn.error, gunicorn.access + +[handlers] +keys=console, error, access + +[formatters] +keys=generic, error, access + +[logger_root] +level=INFO +handlers=console +qualname=root + +[logger_gunicorn.error] +level=INFO +handlers=console +qualname=gunicorn.error + +[logger_gunicorn.access] +level=INFO +handlers=access +qualname=gunicorn.access + +[handler_console] +class=StreamHandler +formatter=generic +args=(sys.stdout, ) + +[handler_error] +class=StreamHandler +formatter=error +args=(sys.stdout, ) + +[handler_access] +class=StreamHandler +formatter=access +args=(sys.stdout, ) + +[formatter_generic] +format=%(asctime)s [%(name)s %(levelname)s %(process)d] %(message)s +datefmt=%Y-%m-%d %H:%M:%S %Z +class=logging.Formatter + +[formatter_error] +format=%(asctime)s [%(name)s %(levelname)s %(process)d] %(message)s | %(funcName)s() | %(pathname)s L%(lineno)-4d +datefmt=%Y-%m-%d %H:%M:%S %Z +class=logging.Formatter + +[formatter_access] +format=%(asctime)s [%(name)s %(levelname)s %(process)d] %(message)s +datefmt=%Y-%m-%d %H:%M:%S %Z +class=logging.Formatter diff --git a/brewman/pyproject.toml b/brewman/pyproject.toml index 24fcc089..4068011c 100644 --- a/brewman/pyproject.toml +++ b/brewman/pyproject.toml @@ -6,22 +6,23 @@ authors = ["tanshu "] [tool.poetry.dependencies] python = "^3.11" -uvicorn = {extras = ["standard"], version = "^0.21.1"} -fastapi = {extras = ["all"], version = "^0.100.0"} +uvicorn = {extras = ["standard"], version = "^0.23.2"} +fastapi = {extras = ["all"], version = "^0.101.0"} python-jose = {extras = ["cryptography"], version = "^3.3.0"} passlib = {extras = ["bcrypt"], version = "^1.7.4"} -psycopg2-binary = "^2.9.5" -SQLAlchemy = "^2.0.7" +psycopg2-binary = "^2.9.7" +SQLAlchemy = "^2.0.19" python-multipart = "^0.0.6" PyJWT = "^2.8.0" -alembic = "^1.11.1" +alembic = "^1.11.2" itsdangerous = "^2.1.2" python-dotenv = "^1.0.0" -pydantic = {extras = ["dotenv"], version = "^2.0.3"} +pydantic = {extras = ["dotenv"], version = "^2.1.1"} starlette = "^0.27.0" pandas = "^2.0.0" arq = "^0.25.0" openpyxl = "^3.1.2" +gunicorn = "^21.2.0" [tool.poetry.group.dev.dependencies] flake8 = "^6.0.0" diff --git a/brewman/run.sh b/brewman/run.sh new file mode 100755 index 00000000..8df6f72d --- /dev/null +++ b/brewman/run.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +set -euo pipefail +gunicorn brewman.main:app --worker-class uvicorn.workers.UvicornWorker --config ./gunicorn.conf.py --log-config ./logging.conf \ No newline at end of file diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index 8f707983..e83580ce 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -50,4 +50,4 @@ RUN chmod 777 /app/docker-entrypoint.sh \ && ln -s /app/docker-entrypoint.sh / ENTRYPOINT ["docker-entrypoint.sh"] -CMD ["poetry", "run", "python", "-m", "brewman"] +CMD ["poetry", "run", "gunicorn", "brewman.main:app", "--worker-class", "uvicorn.workers.UvicornWorker", "--config", "/app/gunicorn.conf.py", "--log-config", "/app/logging.conf"]