From 6d0376e09ebea028fe9950c5794358c104c0d2c2 Mon Sep 17 00:00:00 2001 From: jsdio Date: Sun, 15 Oct 2023 23:15:39 +0300 Subject: [PATCH] feat: [#60] review fix --- .env.example | 51 +++++++++++++++++++ README.md | 7 +-- docker-compose.yml | 20 ++++---- src/assistant/Dockerfile | 5 ++ .../inline.py => assistant/data/.gitkeep} | 0 src/assistant/docker-compose.dev.yml | 7 +-- src/assistant/docker-compose.tests.yml | 1 - src/assistant/docker-compose.yml | 1 - src/assistant/entrypoint.sh | 4 +- .../agent/repositories/openai_functions.py | 5 +- .../api/v1/handlers/voice_responce_handler.py | 1 - src/assistant/lib/app/app.py | 4 +- src/assistant/lib/app/settings.py | 1 + .../lib/app/split_settings/__init__.py | 2 + src/assistant/lib/app/split_settings/agent.py | 14 +++++ src/bot_aiogram/tgbot/keyboards/reply.py | 0 src/bot_aiogram/tgbot/misc/states.py | 0 src/template/Dockerfile | 4 +- src/template/entrypoint.sh | 2 - 19 files changed, 98 insertions(+), 31 deletions(-) create mode 100644 .env.example rename src/{bot_aiogram/tgbot/keyboards/inline.py => assistant/data/.gitkeep} (100%) create mode 100644 src/assistant/lib/app/split_settings/agent.py delete mode 100644 src/bot_aiogram/tgbot/keyboards/reply.py delete mode 100644 src/bot_aiogram/tgbot/misc/states.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..77aa7e2 --- /dev/null +++ b/.env.example @@ -0,0 +1,51 @@ +# API +POSTGRES_DRIVER=postgresql+asyncpg +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +POSTGRES_USER=app +POSTGRES_PASSWORD=123qwe +POSTGRES_DB_NAME=movies_database + +NGINX_PORT=80 + +API_HOST=0.0.0.0 +API_PORT=8000 +API_PROTOCOL=http +API_URL=api + +JWT_SECRET_KEY=v9LctjUWwol4XbvczPiLFMDtZ8aal7mm +JWT_ALGORITHM=HS256 + +APP_RELOAD=True + +VOICE_AVAILABLE_FORMATS=mp3,ogg,wav,oga +VOICE_MAX_INPUT_SIZE=5120 +VOICE_MAX_INPUT_SECONDS=30 + +OPENAI_API_KEY=sk-123456789 +OPENAI_STT_MODEL=whisper-1 + +PROXY_HOST=213.166.72.28 +PROXY_PORT=9185 +PROXY_USER=RUnj7n +PROXY_PASSWORD=GPTWgk +PROXY_ENABLE=True + +TTS_YANDEX_API_KEY=SfdSDFSDFSDF +TTS_YANDEX_AUDIO_FORMAT=oggopus +TTS_YANDEX_SAMPLE_RATE_HERTZ=48000 + +TTS_ELEVEN_LABS_API_KEY=SDFSFsdfsdf +TTS_ELEVEN_LABS_DEFAULT_VOICE_ID=EXAVITQu4vr4xnSDxMaL + + +# TGBOT +BOT_CONTAINER_NAME=bot_container_name +BOT_IMAGE_NAME=botimage_name +BOT_NAME=mybotname +BOT_TOKEN=1234567890:DSFsdfsdfsdfSDf +BOT_ADMINS=1234567890 + + +REDIS_HOST=redis +REDIS_PORT=6379 diff --git a/README.md b/README.md index 44bf425..0c239df 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ 1. Скачать [файл базы данных](https://disk.yandex.ru/d/ZAKDDg8lP9DHBQ) с `embeddings` и поместить её по пути `src/assistant/data/dump.sql`. -2. В директории `src/assistant` файл `.env.example` переименовать в `.env` и заполнить переменные окружения. +2. В файл `.env.example` переименовать в `.env` и заполнить переменные окружения. Пример заполнения переменных окружения: ``` @@ -68,12 +68,7 @@ TTS_YANDEX_SAMPLE_RATE_HERTZ=48000 # Частота дискретизации TTS_ELEVEN_LABS_API_KEY=1234567890 # API-ключ ElevenLabs TTS_ELEVEN_LABS_DEFAULT_VOICE_ID=EXAVITQu4vr4xnSDxMaL # ID голоса по умолчанию -``` -3. В директории `src/bot_aiogram` файл `.env.example` переименовать в `.env` и заполнить переменные окружения. - Пример заполнения переменных окружения: - -``` BOT_CONTAINER_NAME=bot_container_name # Название контейнера BOT_IMAGE_NAME=botimage_name # Название образа BOT_NAME=mybotname # Название бота diff --git a/docker-compose.yml b/docker-compose.yml index ed9d75a..61bf936 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,25 +7,24 @@ services: environment: POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: ${POSTGRES_NAME} + POSTGRES_DB: ${POSTGRES_DB_NAME} env_file: - - ./src/assistant/.env + - .env expose: - "${POSTGRES_PORT}" volumes: - - - ./src/assistant/dump.sql:/docker-entrypoint-initdb.d/dump.sql:ro + - ./src/assistant/data/dump.sql:/docker-entrypoint-initdb.d/dump.sql:ro networks: - backend_network api: build: - context: . + context: ./src/assistant/ container_name: api image: fastapi_app restart: always - entrypoint: ["/opt/app/entrypoint.sh"] env_file: - - ./src/assistant/.env + - .env expose: - "${API_PORT}" depends_on: @@ -52,20 +51,21 @@ services: image: "${BOT_IMAGE_NAME:-tg_bot-image}" container_name: "${BOT_CONTAINER_NAME:-tg_bot-container}" build: - context: ./src/bot_aiogram/Dockerfile + context: ./src/bot_aiogram/ restart: always env_file: - - ./src/bot_aiogram/.env + - .env networks: - backend_network - api_network - depends_on: api + depends_on: + - api redis: image: redis:7.0.11 restart: always env_file: - - ./src/bot_aiogram/.env + - .env ports: - "127.0.0.1:${REDIS_PORT}:${REDIS_PORT}" command: redis-server --bind 0.0.0.0 --appendonly yes diff --git a/src/assistant/Dockerfile b/src/assistant/Dockerfile index 63a6bd7..412059b 100644 --- a/src/assistant/Dockerfile +++ b/src/assistant/Dockerfile @@ -18,6 +18,11 @@ RUN pip install poetry \ COPY bin /opt/app/bin COPY lib /opt/app/lib + +COPY alembic /opt/app/alembic +COPY alembic.ini /opt/app/alembic.ini + COPY entrypoint.sh /opt/app/entrypoint.sh RUN chmod +x /opt/app/entrypoint.sh +ENTRYPOINT ["/opt/app/entrypoint.sh"] diff --git a/src/bot_aiogram/tgbot/keyboards/inline.py b/src/assistant/data/.gitkeep similarity index 100% rename from src/bot_aiogram/tgbot/keyboards/inline.py rename to src/assistant/data/.gitkeep diff --git a/src/assistant/docker-compose.dev.yml b/src/assistant/docker-compose.dev.yml index 4305528..00a0e92 100644 --- a/src/assistant/docker-compose.dev.yml +++ b/src/assistant/docker-compose.dev.yml @@ -2,7 +2,7 @@ version: "3" services: postgres: - image: postgres:15.2 + image: ankane/pgvector:v0.5.0 restart: always environment: POSTGRES_USER: ${POSTGRES_USER} @@ -13,6 +13,7 @@ services: ports: - "${POSTGRES_PORT}:${POSTGRES_PORT}" volumes: + - ./dump.sql:/docker-entrypoint-initdb.d/dump.sql:ro - postgres_data:/var/lib/postgresql/data/ networks: - backend_network @@ -23,7 +24,6 @@ services: container_name: fastapi_app image: fastapi_app restart: always - entrypoint: ["/opt/app/entrypoint.sh"] env_file: - .env ports: @@ -48,9 +48,6 @@ services: networks: - api_network -volumes: - postgres_data: - networks: api_network: driver: bridge diff --git a/src/assistant/docker-compose.tests.yml b/src/assistant/docker-compose.tests.yml index 0309e55..1bc976e 100644 --- a/src/assistant/docker-compose.tests.yml +++ b/src/assistant/docker-compose.tests.yml @@ -23,7 +23,6 @@ services: container_name: api image: fastapi_app restart: always - entrypoint: ["/opt/app/entrypoint.sh"] env_file: - .env expose: diff --git a/src/assistant/docker-compose.yml b/src/assistant/docker-compose.yml index d46c390..6ee3e69 100644 --- a/src/assistant/docker-compose.yml +++ b/src/assistant/docker-compose.yml @@ -23,7 +23,6 @@ services: container_name: api image: fastapi_app restart: always - entrypoint: ["/opt/app/entrypoint.sh"] env_file: - .env expose: diff --git a/src/assistant/entrypoint.sh b/src/assistant/entrypoint.sh index e48a4fb..1160f77 100644 --- a/src/assistant/entrypoint.sh +++ b/src/assistant/entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/bash -while ! nc -z postgres 5432; do sleep 1; done; +while ! nc -z $POSTGRES_HOST $POSTGRES_PORT; do sleep 1; done; -alembic upgrade head +poetry run alembic upgrade head exec .venv/bin/python -m bin diff --git a/src/assistant/lib/agent/repositories/openai_functions.py b/src/assistant/lib/agent/repositories/openai_functions.py index c487a77..843a093 100644 --- a/src/assistant/lib/agent/repositories/openai_functions.py +++ b/src/assistant/lib/agent/repositories/openai_functions.py @@ -7,6 +7,7 @@ import sqlalchemy.ext.asyncio as sa_asyncio import lib.agent.repositories as repositories import lib.models as models import lib.orm_models as orm_models +import lib.app.settings as app_settings class OpenAIFunctions: @@ -16,10 +17,12 @@ class OpenAIFunctions: self, repository: repositories.EmbeddingRepository, pg_async_session: sa_asyncio.async_sessionmaker[sa_asyncio.AsyncSession], + settings: app_settings.Settings, ) -> None: self.logger = logging.getLogger(__name__) self.pg_async_session = pg_async_session self.repository = repository + self.settings = settings async def get_movie_by_description(self, description: str) -> list[models.Movie] | None: """Use this function to find data about a movie by movie's description.""" @@ -32,7 +35,7 @@ class OpenAIFunctions: stmt = ( sa.select(orm_models.FilmWork) .order_by(orm_models.FilmWork.embeddings.cosine_distance(embedded_description.root)) - .limit(5) + .limit(self.settings.agent.embeddings_limit) ) response = await session.execute(stmt) neighbours = response.scalars() diff --git a/src/assistant/lib/api/v1/handlers/voice_responce_handler.py b/src/assistant/lib/api/v1/handlers/voice_responce_handler.py index c8c2121..0efac75 100644 --- a/src/assistant/lib/api/v1/handlers/voice_responce_handler.py +++ b/src/assistant/lib/api/v1/handlers/voice_responce_handler.py @@ -47,4 +47,3 @@ class VoiceResponseHandler: ) ) return fastapi.responses.StreamingResponse(io.BytesIO(response.audio_content), media_type="audio/ogg") - # return fastapi.responses.StreamingResponse(io.BytesIO(voice), media_type="audio/ogg") diff --git a/src/assistant/lib/app/app.py b/src/assistant/lib/app/app.py index 1310c2b..94ec467 100644 --- a/src/assistant/lib/app/app.py +++ b/src/assistant/lib/app/app.py @@ -97,7 +97,9 @@ class Application: ) embedding_repository = agent_repositories.EmbeddingRepository(settings=settings) agent_tools = agent_functions.OpenAIFunctions( - repository=embedding_repository, pg_async_session=postgres_client.get_async_session() + repository=embedding_repository, + pg_async_session=postgres_client.get_async_session(), + settings=settings ) tts_yandex_repository = tts.TTSYandexRepository( tts_settings=app_split_settings.TTSYandexSettings(), diff --git a/src/assistant/lib/app/settings.py b/src/assistant/lib/app/settings.py index b17c92f..341d8a1 100644 --- a/src/assistant/lib/app/settings.py +++ b/src/assistant/lib/app/settings.py @@ -4,6 +4,7 @@ import lib.app.split_settings as app_split_settings class Settings(pydantic_settings.BaseSettings): + agent: app_split_settings.AgentSettings = app_split_settings.AgentSettings() api: app_split_settings.ApiSettings = app_split_settings.ApiSettings() app: app_split_settings.AppSettings = app_split_settings.AppSettings() postgres: app_split_settings.PostgresSettings = app_split_settings.PostgresSettings() diff --git a/src/assistant/lib/app/split_settings/__init__.py b/src/assistant/lib/app/split_settings/__init__.py index b7b24df..069d477 100644 --- a/src/assistant/lib/app/split_settings/__init__.py +++ b/src/assistant/lib/app/split_settings/__init__.py @@ -1,3 +1,4 @@ +from .agent import * from .api import * from .app import * from .logger import * @@ -9,6 +10,7 @@ from .tts import * from .voice import * __all__ = [ + "AgentSettings", "ApiSettings", "AppSettings", "LoggingSettings", diff --git a/src/assistant/lib/app/split_settings/agent.py b/src/assistant/lib/app/split_settings/agent.py new file mode 100644 index 0000000..87ca6b3 --- /dev/null +++ b/src/assistant/lib/app/split_settings/agent.py @@ -0,0 +1,14 @@ +import pydantic_settings + +import lib.app.split_settings.utils as app_split_settings_utils + + +class AgentSettings(pydantic_settings.BaseSettings): + model_config = pydantic_settings.SettingsConfigDict( + env_file=app_split_settings_utils.ENV_PATH, + env_prefix="AGENT_", + env_file_encoding="utf-8", + extra="ignore", + ) + + embeddings_limit: int = 5 diff --git a/src/bot_aiogram/tgbot/keyboards/reply.py b/src/bot_aiogram/tgbot/keyboards/reply.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/bot_aiogram/tgbot/misc/states.py b/src/bot_aiogram/tgbot/misc/states.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/template/Dockerfile b/src/template/Dockerfile index 202c1a7..38363b3 100644 --- a/src/template/Dockerfile +++ b/src/template/Dockerfile @@ -13,7 +13,7 @@ COPY poetry.toml /opt/app/poetry.toml WORKDIR /opt/app -RUN pip install poetry \ +RUN pip install poetry \ && poetry install --no-dev COPY bin /opt/app/bin @@ -21,3 +21,5 @@ COPY lib /opt/app/lib COPY entrypoint.sh /opt/app/entrypoint.sh RUN chmod +x /opt/app/entrypoint.sh + +CMD [".venv/bin/python", "-m", "bin"] diff --git a/src/template/entrypoint.sh b/src/template/entrypoint.sh index fe288b7..615cd9e 100644 --- a/src/template/entrypoint.sh +++ b/src/template/entrypoint.sh @@ -1,5 +1,3 @@ #!/bin/bash while ! nc -z postgres 5432; do sleep 1; done; - -exec .venv/bin/python -m bin \ No newline at end of file