diff --git a/.devcontainer.json b/.devcontainer.json new file mode 100644 index 0000000000000000000000000000000000000000..372f017809f95765796d65e608c07ef9a4d440ea --- /dev/null +++ b/.devcontainer.json @@ -0,0 +1,14 @@ +// See https://aka.ms/vscode-remote/devcontainer.json for format details. +{ + "dockerComposeFile": ["docker-compose.yml", "docker-compose.dev.yml"], + "service": "backend", + "workspaceFolder": "/code/", + "extensions": ["ms-python.python", + "ms-vscode.atom-keybindings", + "mrorz.language-gettext", + "ms-python.vscode-pylance", + "batisteo.vscode-django", + "keno.uikit-3-snippets", + "dbaeumer.vscode-eslint", + "christian-kohler.npm-intellisense"] +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..3e4e90f35c2d1f35ab479bff3bd9a9509e800429 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.gitignore +Dockerfile +Makefile +README.md +screenshots/ +tags +media/ diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000000000000000000000000000000000000..e058777ae1af2acb500275c2576005fb36036716 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,78 @@ +name: Docker + +on: + push: + # Publish `dev` as Docker `latest` image. + branches: + - dev + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + +env: + IMAGE_NAME: librephotos + +jobs: + # Run tests. + # See also https://docs.docker.com/docker-hub/builds/automated-testing/ + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run tests + run: | + if [ -f docker-compose.test.yml ]; then + docker-compose --file docker-compose.test.yml build + docker-compose --file docker-compose.test.yml run sut + else + DOCKER_BUILDKIT=1 docker build . --file Dockerfile + fi + + # Push image to GitHub Packages. + # See also https://docs.docker.com/docker-hub/builds/ + push: + # Ensure test job passes before pushing image. + needs: test + + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@v2 + + - name: Build image + run: DOCKER_BUILDKIT=1 docker build . --file Dockerfile --tag $IMAGE_NAME + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Push image + run: | + IMAGE_ID=reallibrephotos/$IMAGE_NAME + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + + # Use Docker `dev` tag + VERSION=dev + + echo IMAGE_ID=$IMAGE_ID + echo VERSION=$VERSION + + docker tag $IMAGE_NAME $IMAGE_ID:$VERSION + docker push $IMAGE_ID:$VERSION diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..1841d63ea137a945d2ae9d09b5d786b560247cf1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,172 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# visual studio +.vs/ + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# LibrePhotos +densecap/data/models/densecap/densecap-pretrained-vgg16.t7 +*/*.pkl +*/*/*.pkl +thumbnails +media +samplephotos +tags +api/places365/model/ +Conv2d.patch +Linear.patch +Sequential.patch +BatchNorm2d.patch +AvgPool2d.patch +ReLU.patch +run_docker.sh +logs/ +playground +api/im2txt/data/ +api/im2txt/models/ +api/im2txt/png/ +*.ipynb +api/im2txt/*.tar.gz +api/places365/*.tar.gz +*.db +media* +protected_media +librephotos +vscode/server-extensions +vscode/server-insiders-extensions diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..bd72eb8200c41241dd121c2071d3e406cbd7c69c --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +.PHONY: default build run shell + +DOCKER_TAG ?= ownphotos-backend + +default: build + +build: + docker build -t $(DOCKER_TAG) . + +run: build + docker run $(DOCKER_TAG) + +shell: build + docker run --rm -it $(DOCKER_TAG) /bin/bash diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4ce562af942e63f997bac97c4eb087be4d7f8ac9 --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +# For common users: +Set the needed variables in .env (take librephotos.env as model) + +Clone the repo: `git clone git@github.com:LibrePhotos/librephotos-docker.git` + +docker-compose up -d +This will get the pre-built images and start all the needed processes + +# For developers: +Set the needed variables in .evn (take librephotos.env as model) +Also set the codedir variable that tells docker where your source folder are +Pull frontend code with `git clone https://github.com/LibrePhotos/librephotos-frontend.git ${codedir}/frontend/` + +Pull backend code into `git clone https://github.com/LibrePhotos/librephotos.git ${codedir}/backend/` + +Pull this repo and run `docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d` + +This will build images from scratch (can take a long time the first time) +Now you can develop and benefit from code auto reload from both backend and frontend +N.B. If you already built this image once, when you do force rebuild you have to run + +`docker-compose -f docker-compose.yml -f docker-compose.dev.yml build --no-cache frontend` + +`docker-compose -f docker-compose.yml -f docker-compose.dev.yml build --no-cache backend` + +based on which one you changed if these changes need rebuild as for added dependencies/libraries etc. + +and then the usual `docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d` + +If you use vscode you can also benefit auto completion and other goodies. For this just type `code .` in your dockerfile folder. +Vscode will open and ask you if you want to reopen it in the container. If you do it he will add his server to the docker layers (first time you do it can take a couple of minutes) and you will have the same python interpreter librephotos is using internally hence the same installed libraries in auto completion and the same development environment will be shared by all devs (isort, flake8 and pylint for example) diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..8c8b02cec8d4bb4ce31f148a09f8a38f8d3d0e77 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.8 + +# system packages installation +RUN apt update && apt install -y curl libopenblas-dev libmagic1 libboost-all-dev libxrender-dev liblapack-dev git bzip2 cmake build-essential libsm6 libglib2.0-0 libgl1-mesa-glx --no-install-recommends + +# pre trained models download +WORKDIR /data_models +RUN mkdir -p /data_models/places365/ +RUN mkdir -p /data_models/im2txt/ +RUN mkdir -p /root/.cache/torch/hub/checkpoints/ +RUN curl -SL https://s3.eu-central-1.amazonaws.com/ownphotos-deploy/places365_model.tar.gz | tar -zxC /data_models/places365/ +RUN curl -SL https://s3.eu-central-1.amazonaws.com/ownphotos-deploy/im2txt_model.tar.gz | tar -zxC /data_models/im2txt/ +RUN curl -SL https://s3.eu-central-1.amazonaws.com/ownphotos-deploy/im2txt_data.tar.gz | tar -zxC /data_models/im2txt/ +RUN curl -SL https://download.pytorch.org/models/resnet152-b121ed2d.pth -o /root/.cache/torch/hub/checkpoints/resnet152-b121ed2d.pth +RUN pip install torch==1.7.1+cpu torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html +RUN pip install -v --install-option="--no" --install-option="DLIB_USE_CUDA" --install-option="--no" --install-option="USE_AVX_INSTRUCTIONS" --install-option="--no" --install-option="USE_SSE4_INSTRUCTIONS" dlib +# actual project +ARG DEBUG +WORKDIR /code +RUN git clone https://github.com/LibrePhotos/librephotos . +RUN pip install -r requirements.txt +RUN if [ "$DEBUG" = 1 ] ; then pip install -r requirements.dev.txt ; fi +RUN python -m spacy download en_core_web_sm +EXPOSE 8001 + +COPY entrypoint.sh /entrypoint.sh +CMD ["/entrypoint.sh"] diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh new file mode 100755 index 0000000000000000000000000000000000000000..b97aaa0b68550818cc8aeef3cebeccc63b0766c8 --- /dev/null +++ b/backend/entrypoint.sh @@ -0,0 +1,24 @@ +#! /bin/bash +export PYTHONUNBUFFERED=TRUE +mkdir -p /logs + +python image_similarity/main.py 2>&1 | tee /logs/gunicorn_image_similarity.log & +python manage.py showmigrations | tee /logs/show_migrate.log +python manage.py migrate | tee /logs/command_migrate.log +python manage.py showmigrations | tee /logs/show_migrate.log +python manage.py build_similarity_index 2>&1 | tee /logs/command_build_similarity_index.log +python manage.py clear_cache +python manage.py createadmin -u $ADMIN_USERNAME $ADMIN_EMAIL 2>&1 | tee /logs/command_createadmin.log + +echo "Running backend server..." + +python manage.py rqworker default 2>&1 | tee /logs/rqworker.log & + +if [ "$DEBUG" = 1 ] +then + echo "develompent backend starting" + gunicorn --worker-class=gevent --timeout 36000 --reload --bind backend:8001 --log-level=info ownphotos.wsgi 2>&1 | tee /logs/gunicorn_django.log +else + echo "production backend starting" + gunicorn --worker-class=gevent --timeout 3600 --bind backend:8001 --log-level=info ownphotos.wsgi 2>&1 | tee /logs/gunicorn_django.log +fi diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000000000000000000000000000000000000..c38b0c2bd77b36375d90c524cf19430e727c98fd --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,70 @@ +# Run options: +# 1. There are no options - This add additional tools to aid in the development of Libre Photos +# run cmd: docker-compose up -f docker-compose.yml -f docker-compose.dev.yml -d +# 2. Current added tools: +# pgadmin User admin@admin pass admin port 3001 + +# DO NOT EDIT +# The .env file has everything you need to edit. +# Run options: +# 1. Use prebuilt images (preferred method): +# run cmd: docker-compose up -d +# 2. Build images on your own machine: +# build cmd: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build +# run cmd: docker-compose up -d + +version: '3.8' +services: + + proxy: + tty: true + build: + context: ./proxy + dockerfile: Dockerfile + volumes: + - ${myPhotos}:/data + - ${proMedia}:/protected_media + - ./proxy/nginx.conf:/etc/nginx/nginx.conf:ro + + frontend: + tty: true + environment: + - DEBUG=1 + build: + context: ./frontend + dockerfile: Dockerfile + volumes: + - ${codedir}/librephotos-frontend:/usr/src/app + - ./frontend/entrypoint.sh:/entrypoint.sh + + backend: + tty: true + stdin_open: true + environment: + - DEBUG=1 + build: + context: ./backend + dockerfile: Dockerfile + args: + DEBUG: 1 + volumes: + - ${myPhotos}:/data + - ${proMedia}:/protected_media + - ${logLocation}:/logs + - ${codedir}/librephotos:/code + - ${cachedir}:/root/.cache + - ./vscode/server-extensions:/root/.vscode-server/extensions + - ./vscode/server-insiders-extensions:/root/.vscode-server-insiders/extensions + - ./vscode/settings.json:/code/.vscode/settings.json + - ./backend/entrypoint.sh:/entrypoint.sh + + pgadmin: + image: dpage/pgadmin4 + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-admin@admin.com} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:-admin} + volumes: + - $HOME/pgadmin:/root/.pgadmin + ports: + - "3001:80" + restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..6b687fc88f80583f0fab1c5bd761c8f0c60acbf1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,76 @@ +# DO NOT EDIT +# The .env file has everything you need to edit. +# Run options: +# 1. Use prebuilt images (preferred method): +# run cmd: docker-compose up -d +# 2. Build images on your own machine: +# build cmd: COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker-compose build +# run cmd: docker-compose up -d + +version: '3.8' +services: + proxy: + image: reallibrephotos/librephotos-proxy:${tag} + restart: always + volumes: + - ${myPhotos}:/data + - ${proMedia}:/protected_media + ports: + - ${httpPort}:80 + depends_on: + - backend + - frontend + + db: + image: postgres + restart: always + environment: + - POSTGRES_USER=${dbUser} + - POSTGRES_PASSWORD=${dbPass} + - POSTGRES_DB=${dbName} + volumes: + - ${dbLocation}:/var/lib/postgresql/data + command: postgres -c fsync=off -c synchronous_commit=off -c full_page_writes=off -c random_page_cost=1.0 + + frontend: + image: reallibrephotos/librephotos-frontend:${tag} + restart: always + depends_on: + - backend + + backend: + image: reallibrephotos/librephotos:${tag} + restart: always + volumes: + - ${myPhotos}:/data + - ${proMedia}:/protected_media + - ${logLocation}:/logs + - ${cachedir}:/root/.cache + + environment: + - SECRET_KEY=${shhhhKey} + - BACKEND_HOST=backend + - ADMIN_EMAIL=${adminEmail} + - ADMIN_USERNAME=${userName} + - ADMIN_PASSWORD=${userPass} + - DB_BACKEND=postgresql + - DB_NAME=${dbName} + - DB_USER=${dbUser} + - DB_PASS=${dbPass} + - DB_HOST=db + - DB_PORT=5432 + - REDIS_HOST=redis + - REDIS_PORT=6379 + - MAPBOX_API_KEY=${mapApiKey} + - TIME_ZONE=${timeZone} + - WEB_CONCURRENCY=${gunniWorkers} + - SKIP_PATTERNS=${skipPatterns} + - DEBUG=0 + + # Wait for Postgres + depends_on: + - db + + redis: + image: redis + restart: always diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..f9e7a6f083957bbffbced878845e4b6e9db9efaa --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,15 @@ +FROM node + +RUN apt-get update && apt-get install -y curl git xsel + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app +RUN npm install -g serve +ENV CLI_WIDTH 80 +RUN git clone https://github.com/LibrePhotos/librephotos-frontend /usr/src/app +RUN npm install +RUN npm run build + +EXPOSE 3000 +COPY entrypoint.sh /entrypoint.sh +CMD ["/entrypoint.sh"] diff --git a/frontend/entrypoint.sh b/frontend/entrypoint.sh new file mode 100755 index 0000000000000000000000000000000000000000..9bae3789ce32a6328f937c179c220454d46c604e --- /dev/null +++ b/frontend/entrypoint.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +echo "installing frontend" +npm install + +echo "serving frontend" +if [ "$DEBUG" = 1 ] +then + echo "develompent running frontend" + npm run start +else + echo "productions running frontend" + serve build -d -l 3000 +fi +# DANGEROUSLY_DISABLE_HOST_CHECK=true HOST=0.0.0.0 npm start diff --git a/librephotos.env b/librephotos.env new file mode 100644 index 0000000000000000000000000000000000000000..14a44bc9a8da92cc43bd67c55e3b3450b390e348 --- /dev/null +++ b/librephotos.env @@ -0,0 +1,76 @@ +# This file contains all the things you need to change to set up your Libre Photos. +# There are a few items that must be set for it to work such as the location of your photos. +# After the mandatory entry's there are some optional ones that you may set. + +# Start of mandatory changes. + +# Location of your photos. +myPhotos=./librephotos/pictures + +# Comma delimited list of patterns to ignore (e.g. "@eaDir,#recycle" for synology devices) +skipPatterns= + +# The location where you would like to store the log files. The full path must exist as it will not be created for you. +logLocation=./librephotos/logs/ + +# Protected media directory. This is where we store some files like the thumbnails of your images. +proMedia=./librephotos/media + +# Where shall we store the cache files. +cachedir=./librephotos/cache + +# Where shall we store the database. +dbLocation=./librephotos/data + +# Username for the Administrator login. +userName=admin + +# Password for the administrative user you set above. +userPass=admin + +# Email for the administrative user. +adminEmail=admin@example.com + +# Secret Key. Get one here https://rb.gy/emgmwo (Shortened random.org link) +shhhhKey=D2VymuMn2gAhx4tmAawd + +# Time zone https://docs.djangoproject.com/en/3.1/ref/settings/#std:setting-TIME_ZONE +timeZone=America/Denver + +#What port should Libre Photos be accessed at (Default 3000) +httpPort=3000 + +# ------------------------------------------------------------------------------------------------ + +# Wow, we are at the optional now. Pretty easy so far. You do not have to change any of the below. + +# Do you want to see on a map where all your photos where taken (if a location is stored in your photos) +# Get a Map box API Key https://account.mapbox.com/auth/signup/ +mapApiKey= + +# What branch should we install the stable or the development branch (dev) +# NOTE: only the dev branch is currently supported as we are working towards the stable version. +tag=dev + +# You can set the database name. Did you know Libre Photos was forked from OwnPhotos? +dbName=librephotos + +# Here you can change the user name for the database. +dbUser=docker + +# The password used by the database. +dbPass=AaAa1234 + +# This setting can dramatically affect how resource-intensive and the speed of scanning photos +# A positive integer generally in the 2-4 x $(NUM_CORES) range. +# You’ll want to vary this a bit to find the best for your particular workload. +# Each worker needs 800MB of RAM. Change at your own will. Default is 2. +gunniWorkers=2 + +# --------------------------------------------------------------------------------------------- + +# If you are not a developer ignore the following parameters: you will never need them. + +# Where shall we store the backend and frontend code files. +codedir=./librephotos/code + diff --git a/proxy/Dockerfile b/proxy/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..01bad77c488c85a10a29a66a1be048b70fc7e146 --- /dev/null +++ b/proxy/Dockerfile @@ -0,0 +1,2 @@ +FROM nginx +COPY nginx.conf /etc/nginx/nginx.conf diff --git a/proxy/nginx.conf b/proxy/nginx.conf new file mode 100644 index 0000000000000000000000000000000000000000..063e164f2250036743985edc1bf59355ead14c2a --- /dev/null +++ b/proxy/nginx.conf @@ -0,0 +1,45 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log debug; + +events { + worker_connections 1024; +} + +http { + server { + listen 80; + location / { + # React routes are entirely on the App side in the web broswer + # Always proxy to root with the same page request when nginx 404s + error_page 404 /; + proxy_intercept_errors on; + proxy_set_header Host $host; + proxy_pass http://frontend:3000/; + } + location ~ ^/(api|media)/ { + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host backend; + include uwsgi_params; + proxy_pass http://backend:8001; + } + # Django media + location /protected_media { + internal; + alias /protected_media/; + } + + # Original Photos + location /original { + internal; + alias /data/; + } + # Nextcloud Original Photos + location /nextcloud_original { + internal; + alias /data/nextcloud_media/; + } + } +} diff --git a/vscode/settings.json b/vscode/settings.json new file mode 100644 index 0000000000000000000000000000000000000000..31f641cb982cc6794bb3a7993edb4501b68a77c7 --- /dev/null +++ b/vscode/settings.json @@ -0,0 +1,23 @@ +{ + "files.exclude": { + "**/*.py[co]": true, + "**/*.so": true, + "**/__pycache__": true + }, + "python.pythonPath": "/usr/local/bin/python", + "python.linting.enabled": true, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": [ + "--exclude: .+/migrations/", + "--max-line-length=119" + ], + "python.linting.pylintArgs": [ + "--load-plugins=pylint_django", + "-d", + "E0239", + "-d", + "C0111" + ], + "python.linting.pylintEnabled": true, + "python.sortImports.path": "isort" +}