diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index f0268a0d971866f5868d14ec2379877f35fcc069..82bef9fce737f1ee7192b69f077e4f502636bf9b 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -13,7 +13,7 @@ on:
env:
IMAGE_NAME: librephotos
-
+
jobs:
# Run tests.
# See also https://docs.docker.com/docker-hub/builds/automated-testing/
@@ -25,7 +25,7 @@ jobs:
- name: Run tests
run: echo "To-Do Add Running Tests"
-
+
build:
name: Building dev build
# Ensure test job passes before pushing image.
@@ -33,23 +33,23 @@ jobs:
needs: test
runs-on: ubuntu-latest
-
+
steps:
- name: Checkout
uses: actions/checkout@v2
-
+
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
+
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
+
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
+
- name: Build and push
uses: docker/build-push-action@v2
with:
@@ -57,7 +57,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
tags: reallibrephotos/librephotos:dev
-
+
release-job:
name: Releasing
if: github.event_name == 'release' && github.event.action == 'created'
@@ -68,19 +68,19 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
-
+
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
+
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
+
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
+
- name: Build and push
uses: docker/build-push-action@v2
with:
diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
index 2c3a39ebe749dcc1a4e900454d36d2b2e5003ba1..21652f6409c5b88aa1dcb7de4ebf51254f248b56 100644
--- a/.github/workflows/frontend.yml
+++ b/.github/workflows/frontend.yml
@@ -5,7 +5,7 @@ on:
types: [created]
repository_dispatch:
types: frontend-commit-event
-
+
push:
# Publish `dev` as Docker `latest` image.
branches:
@@ -34,7 +34,7 @@ jobs:
- name: Run tests
run: echo "To-Do Add Tests"
-
+
build:
if: github.event_name != 'release'
# Ensure test job passes before pushing image.
@@ -45,19 +45,19 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
-
+
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
+
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
+
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
+
- name: Build and push
uses: docker/build-push-action@v2
with:
@@ -65,7 +65,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
tags: reallibrephotos/librephotos-frontend:dev
-
+
release-job:
name: Releasing
if: github.event_name == 'release' && github.event.action == 'created'
@@ -77,19 +77,19 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
-
+
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
+
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
+
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
+
- name: Build and push
uses: docker/build-push-action@v2
with:
diff --git a/.github/workflows/proxy.yml b/.github/workflows/proxy.yml
index c2a50531e73b387c0733a04f60c782c0460741e6..7ab0c9e0884907a4f422e80f08bdaae5d4ee00b4 100644
--- a/.github/workflows/proxy.yml
+++ b/.github/workflows/proxy.yml
@@ -9,7 +9,7 @@ on:
- main
paths:
- 'proxy/**'
-
+
jobs:
# Run tests.
# See also https://docs.docker.com/docker-hub/builds/automated-testing/
@@ -21,7 +21,7 @@ jobs:
- name: Run tests
run: echo "To-Do Add Tests"
-
+
build:
if: github.event_name != 'release'
# Ensure test job passes before pushing image.
@@ -32,19 +32,19 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
-
+
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
+
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
+
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
+
- name: Build and push
uses: docker/build-push-action@v2
with:
@@ -52,7 +52,9 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
tags: reallibrephotos/librephotos-proxy:dev
-
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
+
release-job:
name: Releasing
if: github.event_name == 'release' && github.event.action == 'created'
@@ -64,19 +66,19 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
-
+
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
-
+
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
+
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-
+
- name: Build and push
uses: docker/build-push-action@v2
with:
diff --git a/backend/dependencies/Dockerfile b/backend/dependencies/Dockerfile
index 531a1bffdc605aa6c6ef950b0ac8084bf94fd927..bd91c1153953511a3be35fc9af80a24cc003e8cb 100644
--- a/backend/dependencies/Dockerfile
+++ b/backend/dependencies/Dockerfile
@@ -1,96 +1,114 @@
-FROM ubuntu:hirsute
+FROM ubuntu:20.04
# system packages installation
-
+ARG TARGETPLATFORM
ENV DEBIAN_FRONTEND=noninteractive
-# Install python
-RUN apt-get update \
- && apt-get install -y python3-pip python3-dev \
- && cd /usr/local/bin \
- && ln -s /usr/bin/python3 python \
- && pip3 install --upgrade pip
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ build-essential \
+ bzip2 \
+ cifs-utils \
+ curl \
+ ffmpeg \
+ git \
+ libboost-all-dev \
+ libcfitsio-dev \
+ libexif-dev \
+ libexpat-dev \
+ libexpat1-dev \
+ libgif-dev \
+ libgl1-mesa-glx \
+ libglib2.0-dev \
+ libgsf-1-dev \
+ libheif-dev \
+ libimage-exiftool-perl \
+ libimagequant-dev \
+ libjpeg-dev \
+ liblapack-dev \
+ liblcms2-dev \
+ libmagic1 \
+ libopenblas-dev \
+ libopenexr-dev \
+ liborc-dev \
+ libpng-dev \
+ libpq-dev \
+ libraw-dev \
+ librsvg2-dev \
+ libsm6 \
+ libtiff5-dev \
+ libtool \
+ libtool-bin \
+ libwebp-dev \
+ libxrender-dev \
+ nfs-common \
+ pkg-config \
+ python3-dev \
+ python3-pip \
+ swig \
+ unzip && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then pip3 install torch torchvision -f https://torch.kmtea.eu/whl/stable.html; else pip3 install torch torchvision; fi
-RUN apt update && apt install -y ffmpeg swig cmake libimage-exiftool-perl curl nfs-common cifs-utils libopenblas-dev libheif-dev libmagic1 libraw-dev libboost-all-dev libxrender-dev liblapack-dev git bzip2 cmake build-essential libsm6 libglib2.0-dev libgl1-mesa-glx libpq-dev libexpat-dev \
- librsvg2-dev \
- libpng-dev \
- libgif-dev \
- libjpeg-dev \
- libexif-dev \
- liblcms2-dev \
- liborc-dev \
- pkg-config \
- libexpat1-dev \
- libtiff5-dev \
- libgsf-1-dev \
- libopenexr-dev \
- libcfitsio-dev \
- libimagequant-dev \
- libtool \
- libtool-bin \
- wget \
- libwebp-dev --no-install-recommends
+RUN pip3 install pyvips==2.1.15 cmake==3.21.2
#Build and install libraw
-RUN git clone https://github.com/LibRaw/LibRaw
-RUN ls
-WORKDIR /LibRaw
-RUN autoreconf --install
-RUN ./configure
-RUN make
-RUN make install
+WORKDIR /tmp/builds
+RUN git clone https://github.com/LibRaw/LibRaw && \
+ cd LibRaw && \
+ autoreconf --install && \
+ ./configure && \
+ make && \
+ make install
#Build and install imagemagick
-WORKDIR /
-RUN mkdir /imagemagick
-RUN curl -SL https://www.imagemagick.org/download/ImageMagick.tar.gz | tar -zxC /imagemagick
-WORKDIR /imagemagick/ImageMagick-7.1.0-5
-RUN ./configure --with-modules
-RUN make install
-RUN ldconfig /usr/local/lib
+WORKDIR /tmp/builds
+ARG IMAGEMAGICK_VERSION=7.1.0-5
+RUN curl -SL https://www.imagemagick.org/download/releases/ImageMagick-${IMAGEMAGICK_VERSION}.tar.gz | tar -zx && \
+ cd ImageMagick-${IMAGEMAGICK_VERSION} && \
+ ./configure --with-modules && \
+ make install && \
+ ldconfig /usr/local/lib
# Build and install libvips
+WORKDIR /tmp/builds
ARG VIPSVERSION=8.11.0
-ARG VIPSURL=https://github.com/libvips/libvips/releases/download
-WORKDIR /usr/local/src
-RUN wget ${VIPSURL}/v${VIPSVERSION}/vips-${VIPSVERSION}.tar.gz \
- && tar xzf vips-${VIPSVERSION}.tar.gz \
+RUN curl -SL https://github.com/libvips/libvips/releases/download/v${VIPSVERSION}/vips-${VIPSVERSION}.tar.gz | tar -xz \
&& cd vips-${VIPSVERSION} \
&& ./configure \
&& make V=0 \
&& make install \
&& ldconfig
-# pre trained models download
-WORKDIR /data_models
-RUN mkdir -p /root/.cache/torch/hub/checkpoints/
-
-RUN curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/places365.tar.gz | tar -zxC /data_models/
-RUN curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/im2txt.tar.gz | tar -zxC /data_models/
-RUN curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/clip-embeddings.tar.gz | tar -zxC /data_models/
-RUN curl -SL https://download.pytorch.org/models/resnet152-b121ed2d.pth -o /root/.cache/torch/hub/checkpoints/resnet152-b121ed2d.pth
-
-RUN pip install torch torchvision
-
-RUN pip3 install pyvips
-
# Build and install dlib
-WORKDIR /
-RUN git clone https://github.com/davisking/dlib.git && \
- mkdir /dlib/build && \
- cd /dlib/build && \
+WORKDIR /tmp/builds
+RUN git clone --depth 1 --branch 'v19.22' https://github.com/davisking/dlib.git && \
+ mkdir dlib/build && \
+ cd dlib/build && \
cmake .. -DDLIB_USE_CUDA=0 -DUSE_AVX_INSTRUCTIONS=0 -DLIB_NO_GUI_SUPPORT=0 && \
cmake --build . && \
- cd /dlib && \
- python setup.py install --no USE_AVX_INSTRUCTIONS --no DLIB_USE_CUDA --no USE_SSE4_INSTRUCTIONS
+ cd /tmp/builds/dlib && \
+ python3 setup.py install --no USE_AVX_INSTRUCTIONS --no DLIB_USE_CUDA --no USE_SSE4_INSTRUCTIONS
#Build and install faiss. Needs to be build for ARM
-WORKDIR /faiss
-RUN git clone https://github.com/facebookresearch/faiss.git /faiss
-RUN cmake -B build . -DCMAKE_BUILD_TYPE=Release -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=ON -DFAISS_OPT_LEVEL=generic
-RUN make -C build -j4 faiss
-RUN make -C build -j4 swigfaiss
-RUN (cd build/faiss/python && python setup.py install)
-# install unzip
-RUN apt install -y unzip
-# unzip faiss to actually installed it...
-RUN unzip /usr/local/lib/python3.9/dist-packages/faiss*.egg -d /usr/local/lib/python3.9/dist-packages/
+WORKDIR /tmp/builds
+RUN git clone --depth 1 --branch 'v1.7.1' https://github.com/facebookresearch/faiss.git && \
+ cd faiss && \
+ cmake -B build . -DCMAKE_BUILD_TYPE=Release -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=ON -DFAISS_OPT_LEVEL=generic && \
+ make -C build -j4 faiss && \
+ make -C build -j4 swigfaiss && \
+ cd build/faiss/python && \
+ python3 setup.py install && \
+ unzip /usr/local/lib/python3.8/dist-packages/faiss*.egg -d /usr/local/lib/python3.8/dist-packages/
+
+# pre trained models download
+WORKDIR /data_models
+
+RUN mkdir -p /root/.cache/torch/hub/checkpoints/ && \
+ curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/places365.tar.gz | tar -zxC /data_models/ && \
+ curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/im2txt.tar.gz | tar -zxC /data_models/ && \
+ curl -SL https://github.com/LibrePhotos/librephotos-docker/releases/download/0.1/clip-embeddings.tar.gz | tar -zxC /data_models/ && \
+ curl -SL https://download.pytorch.org/models/resnet152-b121ed2d.pth -o /root/.cache/torch/hub/checkpoints/resnet152-b121ed2d.pth
+
+RUN rm -fr /tmp/*
diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh
index b9f335faa5ca7c556e64a3ea03e6fc68af219f72..b6d682deaa862063edccb2a757e2e6552a57cb56 100755
--- a/backend/entrypoint.sh
+++ b/backend/entrypoint.sh
@@ -2,7 +2,6 @@
export PYTHONUNBUFFERED=TRUE
export PYTHONFAULTHANDLER=1
mkdir -p /logs
-service statd start
python image_similarity/main.py 2>&1 | tee /logs/gunicorn_image_similarity.log &
python manage.py showmigrations | tee /logs/show_migrate.log
python manage.py migrate | tee /logs/command_migrate.log
diff --git a/docker-compose.yml b/docker-compose.yml
index df1aa11568953172cbbd67ba2b1a5de5f3aab64d..0f654f165bda8c1895d499b240e4d83bdcc2800d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -57,7 +57,7 @@ services:
- DB_NAME=${dbName}
- DB_USER=${dbUser}
- DB_PASS=${dbPass}
- - DB_HOST=db
+ - DB_HOST=${dbHost}
- DB_PORT=5432
- REDIS_HOST=redis
- REDIS_PORT=6379
diff --git a/librephotos.env b/librephotos.env
index 50f0a3f9f7911e53aa1e8cb8c95ffbd0de2aeb64..79b98f2b5b02531679496661c3c7c3930f08278f 100644
--- a/librephotos.env
+++ b/librephotos.env
@@ -31,6 +31,9 @@ userPass=admin
# Email for the administrative user.
adminEmail=admin@example.com
+# Database host. Only change this if you want to use your own existing Postgres server. If using your own server, you can remove the 'db' container from docker-compose.yml
+dbHost=db
+
# Secret Key. Get one here https://rb.gy/emgmwo (Shortened random.org link)
shhhhKey=D2VymuMn2gAhx4tmAawd