From b253a8a7f7ccb8ca7736e462c0e806f2e13b93bd Mon Sep 17 00:00:00 2001 From: Luigi Marini Date: Tue, 3 Feb 2026 15:04:16 -0600 Subject: [PATCH 1/6] Traefik not working with frontend. --- docker-compose.dev.yml | 128 ++++++++++++++++++++++++++++++++-- docker-compose.extractors.yml | 1 + docker-compose.yml | 38 +++++++--- 3 files changed, 151 insertions(+), 16 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index e396d5f35..ded919d65 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -63,19 +63,26 @@ services: - minio3 - minio4 - postgres: - image: postgres:16 + postgres-keycloak: + image: postgres:18.0 networks: - clowder2 + ports: + - "5432:5432" volumes: - postgres_data:/var/lib/postgresql/data environment: POSTGRES_DB: keycloak_dev POSTGRES_USER: keycloak POSTGRES_PASSWORD: password + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U keycloak -d keycloak" ] + interval: 10s + timeout: 5s + retries: 5 keycloak: - image: quay.io/keycloak/keycloak:20.0 + image: quay.io/keycloak/keycloak:26.4.0 volumes: - ./scripts/keycloak/clowder-realm-dev.json:/opt/keycloak/data/import/realm.json:ro - ./scripts/keycloak/clowder-theme/:/opt/keycloak/themes/clowder-theme/:ro @@ -83,20 +90,27 @@ services: - clowder2 command: - start-dev - - --http-relative-path /keycloak +# - --http-enabled=true +# - --hostname=localhost +# - --hostname-strict=false +## - --hostname-strict-https=false +## - --https-port=0 +## - --http-port=8080 + - --http-relative-path=/keycloak - --import-realm environment: KEYCLOAK_ADMIN: admin KEYCLOAK_ADMIN_PASSWORD: admin KC_DB: postgres - KC_DB_URL_HOST: postgres + KC_DB_URL_HOST: postgres-keycloak KC_DB_URL_DATABASE: keycloak_dev KC_DB_USERNAME: keycloak KC_DB_PASSWORD: password ports: - - 8080:8080 + - "8080:8080" depends_on: - - postgres + postgres-keycloak: + condition: service_healthy maildev: image: maildev/maildev @@ -171,6 +185,106 @@ services: - mongo - rabbitmq +# # Extractors +# name-entity-recognition: +# image: socialmediamacroscope/name_entity_recognition_extractor:latest +# environment: +# CLOWDER_VERSION: 2 +# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F +# networks: +# - clowder2 +# restart: unless-stopped +# depends_on: +# - rabbitmq +# +# network-analysis: +# image: socialmediamacroscope/network_analysis_extractor:latest +# environment: +# CLOWDER_VERSION: 2 +# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F +# networks: +# - clowder2 +# restart: unless-stopped +# depends_on: +# - rabbitmq +# +# topic-modeling: +# image: socialmediamacroscope/topic_modeling_extractor:latest +# environment: +# CLOWDER_VERSION: 2 +# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F +# networks: +# - clowder2 +# restart: unless-stopped +# depends_on: +# - rabbitmq +# +# natural-language-preprocessing: +# image: socialmediamacroscope/preprocessing_extractor:latest +# environment: +# CLOWDER_VERSION: 2 +# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F +# networks: +# - clowder2 +# restart: unless-stopped +# depends_on: +# - rabbitmq +# +# sentiment-analysis: +# image: socialmediamacroscope/sentiment_analysis_extractor:latest +# environment: +# CLOWDER_VERSION: 2 +# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F +# networks: +# - clowder2 +# restart: unless-stopped +# depends_on: +# - rabbitmq +# +# image-preview: +# image: clowder/extractors-image-preview:latest +# environment: +# CLOWDER_VERSION: 2 +# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F +# networks: +# - clowder2 +# restart: unless-stopped +# depends_on: +# - rabbitmq +# +# image-metadata: +# image: clowder/extractors-image-metadata:latest +# environment: +# CLOWDER_VERSION: 2 +# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F +# networks: +# - clowder2 +# restart: unless-stopped +# depends_on: +# - rabbitmq +# +# audio-preview: +# image: clowder/extractors-audio-preview:latest +# environment: +# CLOWDER_VERSION: 2 +# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F +# networks: +# - clowder2 +# restart: unless-stopped +# depends_on: +# - rabbitmq +# +# file-digest: +# image: clowder/extractors-digest:latest +# environment: +# CLOWDER_VERSION: 2 +# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F +# networks: +# - clowder2 +# restart: unless-stopped +# depends_on: +# - rabbitmq + networks: clowder2: name: clowder2 diff --git a/docker-compose.extractors.yml b/docker-compose.extractors.yml index 8795c2c63..8451d2b1c 100644 --- a/docker-compose.extractors.yml +++ b/docker-compose.extractors.yml @@ -82,3 +82,4 @@ services: networks: clowder2: + external: true diff --git a/docker-compose.yml b/docker-compose.yml index 2acf997b7..96f07bed8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,8 +21,8 @@ services: restart: unless-stopped command: - --api.insecure=true - - --providers.docker - # - --entryPoints.web.address=:80 + - --providers.docker=true + - --entryPoints.web.address=:80 ports: # The HTTP port - "80:80" @@ -75,7 +75,10 @@ services: labels: - "traefik.enable=true" - "traefik.http.routers.backend.rule=PathPrefix(`/api`)" + - "traefik.http.routers.backend.entrypoints=web" - "traefik.http.routers.swagger.rule=PathPrefix(`/docs`)" + - "traefik.http.routers.swagger.entrypoints=web" + - "traefik.http.routers.swagger.service=backend" - "traefik.http.services.backend.loadbalancer.server.port=80" - "traefik.http.routers.backend.priority=5" @@ -84,6 +87,8 @@ services: restart: unless-stopped build: context: ./frontend + ports: + - "3000:80" networks: - clowder2 depends_on: @@ -94,6 +99,8 @@ services: labels: - "traefik.enable=true" - "traefik.http.routers.frontend.rule=PathPrefix(`/`)" + - "traefik.http.routers.frontend.entrypoints=web" + - "traefik.http.routers.frontend.service=frontend" - "traefik.http.services.frontend.loadbalancer.server.port=80" - "traefik.http.routers.frontend.priority=1" @@ -134,6 +141,8 @@ services: restart: unless-stopped networks: - clowder2 + ports: + - "27017:27017" volumes: - mongo:/data/db @@ -179,20 +188,27 @@ services: - minio3 - minio4 - postgres: - image: postgres:16 + keycloak-postgres: + container_name: keycloak-postgres + hostname: keycloak-postgres + image: postgres:18.0 restart: unless-stopped networks: - clowder2 volumes: - - postgres_data:/var/lib/postgresql/data + - postgres_data:/var/lib/postgresql + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U keycloak -d keycloak" ] + interval: 10s + timeout: 5s + retries: 5 environment: POSTGRES_DB: keycloak_prod POSTGRES_USER: keycloak POSTGRES_PASSWORD: password keycloak: - image: quay.io/keycloak/keycloak:20.0 + image: quay.io/keycloak/keycloak:26.4.0 restart: unless-stopped networks: - clowder2 @@ -201,21 +217,24 @@ services: - ./scripts/keycloak/clowder-theme/:/opt/keycloak/themes/clowder-theme/:ro command: - start-dev - - --http-relative-path /keycloak + - --http-relative-path=/keycloak - --import-realm environment: KEYCLOAK_ADMIN: admin KEYCLOAK_ADMIN_PASSWORD: admin KC_DB: postgres - KC_DB_URL_HOST: postgres + KC_DB_URL_HOST: keycloak-postgres + KC_DB_URL_PORT: "5432" KC_DB_URL_DATABASE: keycloak_prod KC_DB_USERNAME: keycloak KC_DB_PASSWORD: password depends_on: - - postgres + keycloak-postgres: + condition: service_healthy labels: - "traefik.enable=true" - "traefik.http.routers.keycloak.rule=PathPrefix(`/keycloak`)" + - "traefik.http.routers.keycloak.entrypoints=web" - "traefik.http.services.keycloak.loadbalancer.server.port=8080" - "traefik.http.routers.keycloak.priority=10" @@ -272,3 +291,4 @@ volumes: networks: clowder2: + name: clowder2 From bc569462bbe10cd86ede60b4f867e774c19dcce7 Mon Sep 17 00:00:00 2001 From: Luigi Marini Date: Tue, 3 Feb 2026 17:58:29 -0600 Subject: [PATCH 2/6] Upgraded traefik to fix discovery issue. --- docker-compose.yml | 42 +++++++++++------------------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 96f07bed8..7fcafb4ab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,12 +17,12 @@ x-minio-common: &minio-common services: traefik: - image: traefik:v2.5 + image: traefik:v3.6 restart: unless-stopped command: - - --api.insecure=true - - --providers.docker=true - - --entryPoints.web.address=:80 + - "--api.insecure=true" + - "--providers.docker=true" + - "--entrypoints.web.address=:80" ports: # The HTTP port - "80:80" @@ -75,10 +75,7 @@ services: labels: - "traefik.enable=true" - "traefik.http.routers.backend.rule=PathPrefix(`/api`)" - - "traefik.http.routers.backend.entrypoints=web" - "traefik.http.routers.swagger.rule=PathPrefix(`/docs`)" - - "traefik.http.routers.swagger.entrypoints=web" - - "traefik.http.routers.swagger.service=backend" - "traefik.http.services.backend.loadbalancer.server.port=80" - "traefik.http.routers.backend.priority=5" @@ -87,8 +84,6 @@ services: restart: unless-stopped build: context: ./frontend - ports: - - "3000:80" networks: - clowder2 depends_on: @@ -99,8 +94,6 @@ services: labels: - "traefik.enable=true" - "traefik.http.routers.frontend.rule=PathPrefix(`/`)" - - "traefik.http.routers.frontend.entrypoints=web" - - "traefik.http.routers.frontend.service=frontend" - "traefik.http.services.frontend.loadbalancer.server.port=80" - "traefik.http.routers.frontend.priority=1" @@ -141,8 +134,6 @@ services: restart: unless-stopped networks: - clowder2 - ports: - - "27017:27017" volumes: - mongo:/data/db @@ -188,27 +179,20 @@ services: - minio3 - minio4 - keycloak-postgres: - container_name: keycloak-postgres - hostname: keycloak-postgres - image: postgres:18.0 + postgres: + image: postgres:16 restart: unless-stopped networks: - clowder2 volumes: - - postgres_data:/var/lib/postgresql - healthcheck: - test: [ "CMD-SHELL", "pg_isready -U keycloak -d keycloak" ] - interval: 10s - timeout: 5s - retries: 5 + - postgres_data:/var/lib/postgresql/data environment: POSTGRES_DB: keycloak_prod POSTGRES_USER: keycloak POSTGRES_PASSWORD: password keycloak: - image: quay.io/keycloak/keycloak:26.4.0 + image: quay.io/keycloak/keycloak:20.0 restart: unless-stopped networks: - clowder2 @@ -217,24 +201,21 @@ services: - ./scripts/keycloak/clowder-theme/:/opt/keycloak/themes/clowder-theme/:ro command: - start-dev - - --http-relative-path=/keycloak + - --http-relative-path /keycloak - --import-realm environment: KEYCLOAK_ADMIN: admin KEYCLOAK_ADMIN_PASSWORD: admin KC_DB: postgres - KC_DB_URL_HOST: keycloak-postgres - KC_DB_URL_PORT: "5432" + KC_DB_URL_HOST: postgres KC_DB_URL_DATABASE: keycloak_prod KC_DB_USERNAME: keycloak KC_DB_PASSWORD: password depends_on: - keycloak-postgres: - condition: service_healthy + - postgres labels: - "traefik.enable=true" - "traefik.http.routers.keycloak.rule=PathPrefix(`/keycloak`)" - - "traefik.http.routers.keycloak.entrypoints=web" - "traefik.http.services.keycloak.loadbalancer.server.port=8080" - "traefik.http.routers.keycloak.priority=10" @@ -291,4 +272,3 @@ volumes: networks: clowder2: - name: clowder2 From 98523bf7bcb6246b146bdcd95feb5c9239009be7 Mon Sep 17 00:00:00 2001 From: Luigi Marini Date: Tue, 3 Feb 2026 18:11:46 -0600 Subject: [PATCH 3/6] Removed commented out extractors. --- docker-compose.dev.yml | 100 ----------------------------------------- 1 file changed, 100 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index ded919d65..a965c1257 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -185,106 +185,6 @@ services: - mongo - rabbitmq -# # Extractors -# name-entity-recognition: -# image: socialmediamacroscope/name_entity_recognition_extractor:latest -# environment: -# CLOWDER_VERSION: 2 -# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F -# networks: -# - clowder2 -# restart: unless-stopped -# depends_on: -# - rabbitmq -# -# network-analysis: -# image: socialmediamacroscope/network_analysis_extractor:latest -# environment: -# CLOWDER_VERSION: 2 -# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F -# networks: -# - clowder2 -# restart: unless-stopped -# depends_on: -# - rabbitmq -# -# topic-modeling: -# image: socialmediamacroscope/topic_modeling_extractor:latest -# environment: -# CLOWDER_VERSION: 2 -# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F -# networks: -# - clowder2 -# restart: unless-stopped -# depends_on: -# - rabbitmq -# -# natural-language-preprocessing: -# image: socialmediamacroscope/preprocessing_extractor:latest -# environment: -# CLOWDER_VERSION: 2 -# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F -# networks: -# - clowder2 -# restart: unless-stopped -# depends_on: -# - rabbitmq -# -# sentiment-analysis: -# image: socialmediamacroscope/sentiment_analysis_extractor:latest -# environment: -# CLOWDER_VERSION: 2 -# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F -# networks: -# - clowder2 -# restart: unless-stopped -# depends_on: -# - rabbitmq -# -# image-preview: -# image: clowder/extractors-image-preview:latest -# environment: -# CLOWDER_VERSION: 2 -# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F -# networks: -# - clowder2 -# restart: unless-stopped -# depends_on: -# - rabbitmq -# -# image-metadata: -# image: clowder/extractors-image-metadata:latest -# environment: -# CLOWDER_VERSION: 2 -# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F -# networks: -# - clowder2 -# restart: unless-stopped -# depends_on: -# - rabbitmq -# -# audio-preview: -# image: clowder/extractors-audio-preview:latest -# environment: -# CLOWDER_VERSION: 2 -# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F -# networks: -# - clowder2 -# restart: unless-stopped -# depends_on: -# - rabbitmq -# -# file-digest: -# image: clowder/extractors-digest:latest -# environment: -# CLOWDER_VERSION: 2 -# RABBITMQ_URI: amqp://guest:guest@rabbitmq:5672/%2F -# networks: -# - clowder2 -# restart: unless-stopped -# depends_on: -# - rabbitmq - networks: clowder2: name: clowder2 From 895be3b73d748ec6e03efcc6b59c6c113606fdac Mon Sep 17 00:00:00 2001 From: Luigi Marini Date: Wed, 4 Feb 2026 15:43:30 -0600 Subject: [PATCH 4/6] Tests failing. Increasing the wait time for rabbitmq to come online. --- .github/workflows/pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 972e64bc0..b45be7023 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -24,8 +24,8 @@ jobs: run: docker compose -f ../docker-compose.dev.yml up -d # wait for docker containers to come up before running tests - - name: Sleep for 60 seconds - run: sleep 60s + - name: Sleep for 120 seconds + run: sleep 120s shell: bash - name: Run tests From 2e7edbf6dd8a3aa6f0e4e6c69a0a093ad1a75919 Mon Sep 17 00:00:00 2001 From: Luigi Marini Date: Wed, 4 Feb 2026 15:44:13 -0600 Subject: [PATCH 5/6] Don't require ssl in keycloak when running in DEV mode. --- scripts/keycloak/clowder-realm-dev.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/keycloak/clowder-realm-dev.json b/scripts/keycloak/clowder-realm-dev.json index 3d0980a9e..44e9b1a99 100644 --- a/scripts/keycloak/clowder-realm-dev.json +++ b/scripts/keycloak/clowder-realm-dev.json @@ -26,7 +26,7 @@ "oauth2DeviceCodeLifespan": 600, "oauth2DevicePollingInterval": 5, "enabled": true, - "sslRequired": "external", + "sslRequired": "NONE", "registrationAllowed": true, "registrationEmailAsUsername": true, "rememberMe": false, From 996c50ed0b86083bd9140c23369bd06c8218fa82 Mon Sep 17 00:00:00 2001 From: Luigi Marini Date: Wed, 4 Feb 2026 15:55:58 -0600 Subject: [PATCH 6/6] Cleaning postgresql volume to make sure old path is not the issue. --- .github/workflows/pytest.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index b45be7023..dd85a454a 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -20,12 +20,15 @@ jobs: - name: Install the project run: uv sync --all-extras --dev + - name: Clean volumes + run: docker volume rm clowder2_postgres_data || true + - name: Start env run: docker compose -f ../docker-compose.dev.yml up -d # wait for docker containers to come up before running tests - - name: Sleep for 120 seconds - run: sleep 120s + - name: Sleep for 60s seconds + run: sleep 60s shell: bash - name: Run tests