This is an automated email from the ASF dual-hosted git repository.
yasith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/master by this push:
new 3aa17eb88c Monolithic Docker Setup for Apache Airavata with Docker
Compose Support (#554)
3aa17eb88c is described below
commit 3aa17eb88c46c882ba9788bb5e917ad9c1420c72
Author: Yasindu Dissanayake <[email protected]>
AuthorDate: Wed Aug 13 23:30:44 2025 +0530
Monolithic Docker Setup for Apache Airavata with Docker Compose Support
(#554)
* Add Dockerfile
* Add docker run script
* Add docker-compose.ymlz
* Modify GH Actions Workflow
* Fix compose file issue
* Add Properties File for Docker-compose
* Fix Properties file path
* Fix issue on Docker-Startup Script
---
.github/workflows/build-and-publish.yml | 84 ++++--
Dockerfile | 110 +++++++
.../deployment-scripts/airavata-server.properties | 328 +++++++++++++++++++++
dev-tools/deployment-scripts/docker-startup.sh | 214 ++++++++++++++
docker-compose.yml | 113 +++++++
5 files changed, 817 insertions(+), 32 deletions(-)
diff --git a/.github/workflows/build-and-publish.yml
b/.github/workflows/build-and-publish.yml
index 1b48c60f65..41650d2165 100644
--- a/.github/workflows/build-and-publish.yml
+++ b/.github/workflows/build-and-publish.yml
@@ -1,22 +1,34 @@
-name: Maven Docker Build & Push
+name: Build and Push Monolithic Airavata Docker Image
on:
push:
- tags:
- - "v*"
- release:
- types: [published]
+ branches: [ main, master ]
+ tags: [ "v*" ]
+ pull_request:
+ branches: [ main, master ]
+ workflow_dispatch:
env:
- DOCKER_REGISTRY: docker.io
- DOCKER_USERNAME: cybershuttle
- VERSION: ${{ github.ref_name }}
+ REGISTRY: docker.io
+ IMAGE_NAME: cybershuttle-airavata:latest
jobs:
- build:
- runs-on: ubuntu-22.04
+ build-and-push:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v4
+ with:
+ distribution: "temurin"
+ java-version: "17"
+
- name: Install OS dependencies
run: |
sudo apt-get update
@@ -31,34 +43,42 @@ jobs:
make -j$(nproc)
sudo make install
- - name: Set up JDK 17
- uses: actions/setup-java@v4
- with:
- distribution: "temurin"
- java-version: "17"
+ - name: Build with Maven (skip tests)
+ run: mvn clean install -DskipTests
- - name: Checkout code
- uses: actions/checkout@v4
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
- - name: Login to Docker Hub
+ - name: Log in to Docker Hub
uses: docker/login-action@v3
with:
+ registry: ${{ env.REGISTRY }}
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
- - name: Build with Maven (skip tests)
- run: mvn clean install -DskipTests
-
- - name: Build Docker Images
- run: |
- mvn docker:build -pl modules/distribution
+ - name: Extract metadata
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.REGISTRY }}/${{ secrets.DOCKER_HUB_USERNAME }}/${{
env.IMAGE_NAME }}
+ tags: |
+ type=ref,event=branch
+ type=ref,event=pr
+ type=semver,pattern={{version}}
+ type=semver,pattern={{major}}.{{minor}}
+ type=sha,prefix={{branch}}-
- - name: List Docker Images
- run: docker images
+ - name: Build and push Docker image
+ uses: docker/build-push-action@v5
+ with:
+ context: .
+ file: ./Dockerfile
+ push: ${{ github.event_name != 'pull_request' }}
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
+ platforms: linux/amd64,linux/arm64
- - name: Tag and Push Images
- run: |
- for image in email-monitor participant post-wm pre-wm api-server
controller; do
- docker tag airavata/$image
${DOCKER_USERNAME}/airavata-$image:${VERSION}
- docker push ${DOCKER_USERNAME}/airavata-$image:${VERSION}
- done
+ - name: Image digest
+ run: echo ${{ steps.meta.outputs.tags }}
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000000..35b7e32063
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,110 @@
+# =============================================================================
+# Combines all Airavata services into one monolithic container
+# =============================================================================
+
+FROM eclipse-temurin:17-jre-jammy
+
+# Install necessary packages including multitail for log monitoring
+RUN apt-get update && apt-get install -y \
+ curl \
+ netcat-openbsd \
+ multitail \
+ && rm -rf /var/lib/apt/lists/*
+
+# Create non-root user for security
+RUN groupadd -r airavata && useradd -r -g airavata -d /opt/airavata airavata
+
+# Set working directory
+WORKDIR /opt/airavata
+
+# Copy all distribution files
+COPY distribution/apache-airavata-api-server-*.tar.gz ./
+COPY distribution/apache-airavata-agent-service-*.tar.gz ./
+COPY distribution/apache-airavata-research-service-*.tar.gz ./
+COPY distribution/apache-airavata-file-server-*.tar.gz ./
+
+# Extract all services
+RUN tar -xzf apache-airavata-api-server-*.tar.gz && \
+ tar -xzf apache-airavata-agent-service-*.tar.gz && \
+ tar -xzf apache-airavata-research-service-*.tar.gz && \
+ tar -xzf apache-airavata-file-server-*.tar.gz && \
+ rm *.tar.gz
+
+# Rename directories for consistency
+RUN mv apache-airavata-api-server-* apache-airavata-api-server && \
+ mv apache-airavata-agent-service-* apache-airavata-agent-service && \
+ mv apache-airavata-research-service-* apache-airavata-research-service && \
+ mv apache-airavata-file-server-* apache-airavata-file-server
+
+# Create necessary directories
+RUN mkdir -p apache-airavata-api-server/conf \
+ apache-airavata-api-server/logs \
+ apache-airavata-api-server/temp \
+ apache-airavata-api-server/data \
+ apache-airavata-api-server/keystores
+
+# Set environment variables
+ENV AIRAVATA_HOME=/opt/airavata/apache-airavata-api-server
+ENV AIRAVATA_AGENT_HOME=/opt/airavata/apache-airavata-agent-service
+ENV AIRAVATA_RESEARCH_HOME=/opt/airavata/apache-airavata-research-service
+ENV AIRAVATA_FILE_HOME=/opt/airavata/apache-airavata-file-server
+ENV JAVA_HOME=/usr/lib/jvm/temurin-17-jre
+ENV PATH="${JAVA_HOME}/bin:${PATH}"
+
+# JVM tuning for production
+ENV JAVA_OPTS="-server \
+ -Xms1g \
+ -Xmx4g \
+ -XX:+UseG1GC \
+ -XX:MaxGCPauseMillis=200 \
+ -XX:+HeapDumpOnOutOfMemoryError \
+ -XX:HeapDumpPath=/opt/airavata/apache-airavata-api-server/logs \
+ -Djava.security.egd=file:/dev/./urandom \
+ -Dfile.encoding=UTF-8 \
+ -Duser.timezone=UTC"
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
+ CMD curl -f http://localhost:8930/ || exit 1
+
+# sharing registry service
+EXPOSE 7878
+# tunnel service
+EXPOSE 8000
+# tunnel service (ingress)
+EXPOSE 17000
+# file service
+EXPOSE 8050
+# api service
+EXPOSE 8930
+# cred store service
+EXPOSE 8960
+# profile service
+EXPOSE 8962
+# registry service
+EXPOSE 8970
+# agent service (http)
+EXPOSE 18800
+# agent service (gRPC)
+EXPOSE 19900
+# research service (http)
+EXPOSE 18889
+# research service (gRPC)
+EXPOSE 19908
+# monitoring
+EXPOSE 9097
+# rest proxy (commented out as restproxy distribution is not available)
+# EXPOSE 8082
+
+# Copy startup script
+COPY dev-tools/deployment-scripts/docker-startup.sh /opt/airavata/start.sh
+
+# Set ownership
+RUN chown -R airavata:airavata /opt/airavata && \
+ chmod +x /opt/airavata/start.sh
+
+# Switch to non-root user
+USER airavata
+
+# Set entrypoint
+ENTRYPOINT ["/opt/airavata/start.sh"]
diff --git a/dev-tools/deployment-scripts/airavata-server.properties
b/dev-tools/deployment-scripts/airavata-server.properties
new file mode 100644
index 0000000000..505a1d0b6b
--- /dev/null
+++ b/dev-tools/deployment-scripts/airavata-server.properties
@@ -0,0 +1,328 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+############################
+# Development Configuration
+# This file provides working defaults for Docker development environment
+############################
+
+############################
+# Registry Database (MariaDB)
+############################
+registry.jdbc.driver=org.mariadb.jdbc.Driver
+registry.jdbc.url=jdbc:mariadb://mysql:3306/experiment_catalog
+registry.jdbc.user=airavata
+registry.jdbc.password=123456
+registry.jdbc.validationQuery=SELECT 1
+
+############################
+# App Catalog Database (MariaDB)
+############################
+appcatalog.jdbc.driver=org.mariadb.jdbc.Driver
+appcatalog.jdbc.url=jdbc:mariadb://mysql:3306/app_catalog
+appcatalog.jdbc.user=airavata
+appcatalog.jdbc.password=123456
+appcatalog.jdbc.validationQuery=SELECT 1
+
+############################
+# Replica Catalog Database (MariaDB)
+############################
+replicacatalog.jdbc.driver=org.mariadb.jdbc.Driver
+replicacatalog.jdbc.url=jdbc:mariadb://mysql:3306/replica_catalog
+replicacatalog.jdbc.user=airavata
+replicacatalog.jdbc.password=123456
+replicacatalog.jdbc.validationQuery=SELECT 1
+
+############################
+# Workflow Catalog Database (MariaDB)
+############################
+workflowcatalog.jdbc.driver=org.mariadb.jdbc.Driver
+workflowcatalog.jdbc.url=jdbc:mariadb://mysql:3306/workflow_catalog
+workflowcatalog.jdbc.user=airavata
+workflowcatalog.jdbc.password=123456
+workflowcatalog.jdbc.validationQuery=SELECT 1
+
+############################
+# Sharing Catalog Database (MariaDB)
+############################
+sharingcatalog.jdbc.driver=org.mariadb.jdbc.Driver
+sharingcatalog.jdbc.url=jdbc:mariadb://mysql:3306/sharing_catalog
+sharingcatalog.jdbc.user=airavata
+sharingcatalog.jdbc.password=123456
+sharingcatalog.jdbc.validationQuery=SELECT 1
+
+############################
+# Credential Store Database (MariaDB)
+############################
+credential.store.jdbc.driver=org.mariadb.jdbc.Driver
+credential.store.jdbc.url=jdbc:mariadb://mysql:3306/credential_store
+credential.store.jdbc.user=airavata
+credential.store.jdbc.password=123456
+credential.store.jdbc.validationQuery=SELECT 1
+
+############################
+# AMQP (RabbitMQ) Configuration
+############################
+rabbitmq.broker.url=amqp://airavata:airavata@rabbitmq:5672/%2F
+experiment.launch.queue=experiment_launch
+rabbitmq.status.exchange.name=status_exchange
+rabbitmq.process.exchange.name=process_exchange
+rabbitmq.experiment.exchange.name=experiment_exchange
+durable.queue=false
+prefetch.count=200
+
+############################
+# ZooKeeper Configuration
+############################
+zookeeper.server.connection=zookeeper:2181
+zookeeper.timeout=30000
+
+############################
+# Helix Cluster Configuration
+############################
+helix.cluster.name=AiravataCluster
+controller.name=AiravataController
+participant.name=AiravataParticipant
+
+############################
+# Monitoring Configuration
+############################
+controller.monitoring.host=0.0.0.0
+controller.monitoring.port=9094
+participant.monitoring.host=0.0.0.0
+participant.monitoring.port=9096
+
+############################
+# Kafka Configuration
+############################
+kafka.broker.url=kafka:9092
+realtime.monitor.broker.consumer.group=monitor
+realtime.monitor.broker.topic=helix-airavata-mq
+
+job.monitor.broker.consumer.group=MonitoringConsumer
+job.monitor.broker.topic=monitoring-data
+job.monitor.broker.publisher.id=AiravataMonitorPublisher
+job.monitor.email.publisher.id=EmailBasedProducer
+job.monitor.realtime.publisher.id=RealtimeProducer
+
+# Email monitor disabled by default
+# email.based.monitor.host=imap.gmail.com
+# email.based.monitor.store.protocol=imaps
+# email.based.monitor.folder.name=INBOX
+# email.expiration.minutes=60
+# email.based.monitoring.period=10000
+# [email protected]
+# email.based.monitor.password=app_password_here
+
+local.data.location=/tmp/airavata
+
+############################
+# ThriftClientPool Configuration
+############################
+thrift.client.pool.abandoned.removal.enabled=true
+thrift.client.pool.abandoned.removal.logged=true
+thrift.client.pool.idle.time.limit.minutes=1
+thrift.client.pool.max.idle.limit=10
+
+############################
+# API Server Configuration
+############################
+# API Server URLs for development
+api.server.host=localhost
+api.server.port=8930
+
+############################
+# Security Configuration
+############################
+# Development defaults - NOT for production!
+enable.sharing=true
+remote.oauth.authorization.server=https://localhost:8443/oauth2/authorize
+remote.oauth.token.server=https://localhost:8443/oauth2/token
+remote.oauth.callback.url=https://localhost:8443/callback-url
+
+############################
+# File Storage Configuration
+############################
+# Default storage locations for development
+# These will be created automatically in the container
+local.data.location=/tmp/airavata
+airavata.data.location=/opt/airavata/data
+airavata.experiments.dir=/opt/airavata/experiments
+
+############################
+# Development Mode Settings
+############################
+# Enable development features
+development.mode=true
+debug.mode=true
+
+# Disable resource validation for development
+validate.compute.resource=false
+validate.storage.resource=false
+
+############################
+# Logging Configuration
+############################
+# Default log levels for development
+log4j.rootLogger=INFO, console, file
+log4j.logger.org.apache.airavata=DEBUG
+
+############################
+# Gateway Configuration
+############################
+# Default gateway for development testing
+default.gateway.id=dev_gateway
+default.gateway.name=Development Gateway
+default.gateway.admin.username=admin
[email protected]
+
+############################
+# Job Submission Configuration
+############################
+# SSH configurations for development
+ssh.key.passphrase=
+ssh.username=
+ssh.password=
+ssh.strict.host.key.checking=false
+
+############################
+# Workflow Configuration
+############################
+# Enable workflow features
+enable.workflows=true
+workflow.interpreter=airavata
+
+############################
+# Notification Configuration
+############################
+# Email notifications (disabled by default)
+enable.email.notification=false
+# enable.email.notification=true
+# email.server=smtp.gmail.com
+# email.server.port=587
+# [email protected]
+# email.password=your-app-password
+# email.ssl=true
+# [email protected]
+
+############################
+# Parser Configuration
+############################
+# Enable parsers for development
+enable.data.parsers=true
+parser.directory=/opt/airavata/parsers
+
+############################
+# Resource Configuration
+############################
+# Default compute resources for development
+localhost.resource.description=Local machine for development
+localhost.resource.hostname=localhost
+localhost.resource.username=airavata
+
+############################
+# Application Configuration
+############################
+# Default applications for development testing
+enable.application.catalog=true
+application.deployment.directory=/opt/airavata/applications
+
+############################
+# Orchestrator Configuration
+############################
+enable.orchestrator=true
+orchestrator.server.host=localhost
+orchestrator.server.port=8940
+orchestrator.server.name=Airavata_Orchestrator
+
+############################
+# Helix Controller & Participant Settings
+############################
+helix.controller.name=AiravataController
+participant.monitoring.enabled=true
+
+############################
+# Workflow Manager Settings
+############################
+pre.workflow.manager.monitoring.enabled=true
+pre.workflow.manager.monitoring.host=localhost
+post.workflow.manager.monitoring.enabled=true
+post.workflow.manager.monitoring.host=localhost
+
+############################
+# Helix Participant Settings
+############################
+helix.participant.name=AiravataParticipant
+
+############################
+# GFAC Configuration
+############################
+enable.gfac=true
+gfac.server.host=localhost
+gfac.server.port=8950
+gfac.server.name=Airavata_GFAC
+
+############################
+# Registry Configuration
+############################
+registry.server.host=localhost
+registry.server.port=8960
+registry.server.name=Airavata_Registry
+
+############################
+# Credential Store Configuration
+############################
+credential.store.server.host=localhost
+credential.store.server.port=8961
+credential.store.server.name=Airavata_CredentialStore
+
+############################
+# User Profile Configuration
+############################
+profile.service.server.host=localhost
+profile.service.server.port=8962
+profile.service.server.name=Airavata_ProfileService
+
+############################
+# Tenant Profile Configuration
+############################
+tenant.profile.service.server.host=localhost
+tenant.profile.service.server.port=8963
+tenant.profile.service.server.name=Airavata_TenantProfileService
+
+############################
+# IAM Admin Services Configuration
+############################
+iam.admin.services.server.host=localhost
+iam.admin.services.server.port=8964
+iam.admin.services.server.name=Airavata_IamAdminServices
+
+############################
+# Group Manager Configuration
+############################
+group.manager.service.server.host=localhost
+group.manager.service.server.port=8965
+group.manager.service.server.name=Airavata_GroupManagerService
+
+############################
+# Sharing Registry Configuration
+############################
+sharing.registry.server.host=localhost
+sharing.registry.server.port=8966
+sharing.registry.server.name=Airavata_SharingRegistryService
diff --git a/dev-tools/deployment-scripts/docker-startup.sh
b/dev-tools/deployment-scripts/docker-startup.sh
new file mode 100644
index 0000000000..f9a13e86b9
--- /dev/null
+++ b/dev-tools/deployment-scripts/docker-startup.sh
@@ -0,0 +1,214 @@
+#!/bin/bash
+
+set -e
+
+# Set Airavata configuration directory
+export AIRAVATA_CONFIG_DIR=/opt/airavata/vault
+
+echo "🚀 Starting Apache Airavata Monolithic Server..."
+echo "📋 All services included: API Server, Agent Service, Research Service,
File Server"
+echo "📁 Properties file location:
/opt/airavata/vault/airavata-server.properties"
+echo "📁 Configuration directory: $AIRAVATA_CONFIG_DIR"
+echo "📊 All logs will be captured and visible via 'docker logs'"
+
+# Wait for dependencies if environment variables are set
+if [ ! -z "${DB_HOST}" ]; then
+ echo "⏳ Waiting for database at ${DB_HOST}:${DB_PORT:-13306}..."
+ while ! nc -z ${DB_HOST} ${DB_PORT:-13306}; do
+ sleep 2
+ done
+ echo "✅ Database is ready"
+fi
+
+if [ ! -z "${RABBITMQ_HOST}" ]; then
+ echo "⏳ Waiting for RabbitMQ at ${RABBITMQ_HOST}:${RABBITMQ_PORT:-5672}..."
+ while ! nc -z ${RABBITMQ_HOST} ${RABBITMQ_PORT:-5672}; do
+ sleep 2
+ done
+ echo "✅ RabbitMQ is ready"
+fi
+
+if [ ! -z "${ZOOKEEPER_HOST}" ]; then
+ echo "⏳ Waiting for ZooKeeper at
${ZOOKEEPER_HOST}:${ZOOKEEPER_PORT:-2181}..."
+ while ! nc -z ${ZOOKEEPER_HOST} ${ZOOKEEPER_PORT:-2181}; do
+ sleep 2
+ done
+ echo "✅ ZooKeeper is ready"
+fi
+
+# Function to log with timestamp
+log() {
+ echo "$(date '+%Y-%m-%d %H:%M:%S') - $1"
+}
+
+# Function to start a service and capture its logs
+start_service() {
+ local service_name=$1
+ local service_script=$2
+ local log_file=$3
+
+ log "🔄 Starting $service_name..."
+
+ # Start the service in background and capture its output
+ # Set the configuration directory for the service
+ {
+ AIRAVATA_CONFIG_DIR=$AIRAVATA_CONFIG_DIR $service_script -d start 2>&1
+ echo "$(date '+%Y-%m-%d %H:%M:%S') - $service_name started
successfully"
+ } | sed "s/^/[$service_name] /" &
+
+ # Wait a moment for the service to start
+ sleep 3
+
+ # Start monitoring the log file in background
+ if [ -f "$log_file" ]; then
+ tail -f "$log_file" | sed "s/^/[$service_name] /" &
+ fi
+}
+
+# ================================
+# Start the API Server Components
+# ================================
+log "🔧 Starting the API Services..."
+
+cd ${AIRAVATA_HOME}
+
+# Start all API services
+start_service "Orchestrator" "./bin/orchestrator.sh"
"${AIRAVATA_HOME}/logs/orchestrator.log"
+start_service "Controller" "./bin/controller.sh"
"${AIRAVATA_HOME}/logs/controller.log"
+start_service "Participant" "./bin/participant.sh"
"${AIRAVATA_HOME}/logs/participant.log"
+# Email Monitor (disabled by default due to config requirements)
+log "📧 Email Monitor disabled (requires email config)"
+echo "$(date '+%Y-%m-%d %H:%M:%S') - Email Monitor disabled (requires email
config)" | sed "s/^/[Email Monitor] /"
+
+# Realtime Monitor (disabled due to API connection dependency)
+log "⏱️ Realtime Monitor disabled (requires API server running)"
+echo "$(date '+%Y-%m-%d %H:%M:%S') - Realtime Monitor disabled (requires API
server running)" | sed "s/^/[Realtime Monitor] /"
+start_service "Pre-WM" "./bin/pre-wm.sh" "${AIRAVATA_HOME}/logs/pre-wm.log"
+start_service "Post-WM" "./bin/post-wm.sh" "${AIRAVATA_HOME}/logs/post-wm.log"
+
+# ================================
+# Start the Agent Service (Optional)
+# ================================
+log "🤖 Checking Agent Service availability..."
+if [ -f "${AIRAVATA_AGENT_HOME}/bin/agent-service.sh" ] && [ -f
"${AIRAVATA_AGENT_HOME}/lib/airavata-agent-service-*.jar" ]; then
+ cd ${AIRAVATA_AGENT_HOME}
+ start_service "Agent Service" "./bin/agent-service.sh"
"${AIRAVATA_AGENT_HOME}/logs/agent-service.log"
+else
+ log "⚠️ Agent Service not available (optional)"
+ echo "$(date '+%Y-%m-%d %H:%M:%S') - Agent Service not available
(optional)" | sed "s/^/[Agent Service] /"
+fi
+
+# ================================
+# Start the Research Service (Optional)
+# ================================
+log "🔬 Checking Research Service availability..."
+if [ -f "${AIRAVATA_RESEARCH_HOME}/bin/research-service.sh" ] && [ -f
"${AIRAVATA_RESEARCH_HOME}/lib/airavata-research-service-*.jar" ]; then
+ cd ${AIRAVATA_RESEARCH_HOME}
+ start_service "Research Service" "./bin/research-service.sh"
"${AIRAVATA_RESEARCH_HOME}/logs/research-service.log"
+else
+ log "⚠️ Research Service not available (optional)"
+ echo "$(date '+%Y-%m-%d %H:%M:%S') - Research Service not available
(optional)" | sed "s/^/[Research Service] /"
+fi
+
+# ================================
+# Start the File Service (Optional)
+# ================================
+log "📁 Checking File Service availability..."
+if [ -f "${AIRAVATA_FILE_HOME}/bin/file-service.sh" ] && [ -f
"${AIRAVATA_FILE_HOME}/lib/airavata-file-server-*.jar" ]; then
+ cd ${AIRAVATA_FILE_HOME}
+ start_service "File Service" "./bin/file-service.sh"
"${AIRAVATA_FILE_HOME}/logs/file-service.log"
+else
+ log "⚠️ File Service not available (optional)"
+ echo "$(date '+%Y-%m-%d %H:%M:%S') - File Service not available
(optional)" | sed "s/^/[File Service] /"
+fi
+
+# ================================
+# Monitor all logs and keep container running
+# ================================
+log "🎉 All Airavata services started successfully!"
+log "📊 Starting comprehensive log monitoring..."
+
+# Wait a moment for all services to initialize
+sleep 10
+
+# Stream all service logs to docker logs and keep container running
+echo "🚀 All Airavata services are running!"
+echo "📋 Service status:"
+echo " - ZooKeeper: Connected (no more 'airavata.host' errors)"
+echo " - All components started successfully"
+echo ""
+echo "🔍 Streaming all logs to 'docker logs -f airavata-monolithic'"
+echo ""
+
+# Function to monitor log file and stream to stdout with prefix
+monitor_log() {
+ local file=$1
+ local prefix=$2
+ local max_wait=60 # Maximum wait time for log file to appear
+ local waited=0
+
+ echo "[$prefix] Waiting for log file: $file"
+
+ # Wait for log file to be created
+ while [ ! -f "$file" ] && [ $waited -lt $max_wait ]; do
+ sleep 2
+ waited=$((waited + 2))
+ done
+
+ if [ -f "$file" ]; then
+ echo "[$prefix] Found log file, starting to stream..."
+ # Use tail -F to follow file even if it gets rotated or recreated
+ tail -F "$file" 2>/dev/null | while IFS= read -r line; do
+ echo "[$prefix] $line"
+ done &
+ else
+ echo "[$prefix] Log file not found after ${max_wait}s: $file"
+ fi
+}
+
+# Wait a moment for log files to be created
+echo "📊 Waiting for log files to be created..."
+sleep 15
+
+# Start monitoring all service logs
+echo "📊 Starting log monitoring for all services..."
+cd ${AIRAVATA_HOME}
+
+# Monitor core service logs
+monitor_log "${AIRAVATA_HOME}/logs/controller.log" "Controller"
+monitor_log "${AIRAVATA_HOME}/logs/participant.log" "Participant"
+
+# Monitor workflow manager logs if they exist
+monitor_log "${AIRAVATA_HOME}/logs/pre-wm.log" "Pre-WM"
+monitor_log "${AIRAVATA_HOME}/logs/post-wm.log" "Post-WM"
+
+# Monitor additional service logs if they exist
+monitor_log "${AIRAVATA_HOME}/logs/orchestrator.log" "Orchestrator"
+monitor_log "${AIRAVATA_HOME}/logs/email-monitor.log" "Email-Monitor"
+monitor_log "${AIRAVATA_HOME}/logs/realtime-monitor.log" "Realtime-Monitor"
+
+# Monitor optional service logs
+monitor_log "${AIRAVATA_HOME}/logs/agent-service.log" "Agent"
+monitor_log "${AIRAVATA_HOME}/logs/research-service.log" "Research"
+monitor_log "${AIRAVATA_HOME}/logs/file-service.log" "File"
+
+echo "📊 All log monitoring started!"
+echo "🔍 Use 'docker logs -f airavata-monolithic' to view all service logs"
+
+# Keep container running and show periodic status
+while true; do
+ sleep 300 # Check every 5 minutes
+ echo "[Status] $(date): Container active, monitoring logs from all
services"
+
+ # Check if critical processes are still running
+ if ! pgrep -f "controller" > /dev/null; then
+ echo "[WARNING] Controller process not found"
+ fi
+ if ! pgrep -f "participant" > /dev/null; then
+ echo "[WARNING] Participant process not found"
+ fi
+
+ # Show number of running tail processes
+ tail_count=$(pgrep -f "tail -F" | wc -l)
+ echo "[Status] Currently monitoring $tail_count log files"
+done
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000000..2403ca741d
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,113 @@
+services:
+ mysql:
+ image: mariadb:10.11
+ container_name: mariadb
+ environment:
+ MYSQL_ROOT_PASSWORD: rootpass
+ MYSQL_DATABASE: experiment_catalog
+ MYSQL_USER: airavata
+ MYSQL_PASSWORD: 123456
+ ports:
+ - "3306:3306"
+ volumes:
+ - mysql-data:/var/lib/mysql
+ - ./.devcontainer/database_scripts/init:/docker-entrypoint-initdb.d:ro
+ networks:
+ - airavata-network
+
+ rabbitmq:
+ image: rabbitmq:3-management
+ container_name: airavata-rabbitmq
+ environment:
+ RABBITMQ_DEFAULT_USER: airavata
+ RABBITMQ_DEFAULT_PASS: airavata
+ ports:
+ - "5672:5672"
+ - "15672:15672"
+ volumes:
+ - rabbitmq-data:/var/lib/rabbitmq
+ networks:
+ - airavata-network
+
+ zookeeper:
+ image: confluentinc/cp-zookeeper:7.4.0
+ container_name: airavata-zookeeper
+ environment:
+ ZOOKEEPER_CLIENT_PORT: 2181
+ ZOOKEEPER_TICK_TIME: 2000
+ volumes:
+ - zookeeper-data:/var/lib/zookeeper/data
+ - zookeeper-logs:/var/lib/zookeeper/log
+ networks:
+ - airavata-network
+
+ kafka:
+ image: confluentinc/cp-kafka:7.4.0
+ container_name: airavata-kafka
+ depends_on:
+ - zookeeper
+ ports:
+ - "9092:9092"
+ environment:
+ KAFKA_BROKER_ID: 1
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+ KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+ KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+ KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+ volumes:
+ - kafka-data:/var/lib/kafka/data
+ networks:
+ airavata-network:
+ aliases:
+ - airavata.host
+
+ airavata-monolithic:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ container_name: airavata-monolithic
+ ports:
+ - "8930:8930" # Airavata API (Thrift RPC)
+ - "8050:8050" # File Server (Thrift RPC)
+ - "18800:18800" # Agent Service (Thrift RPC)
+ - "18889:18889" # Research Service (Thrift RPC)
+ - "19900:19900" # Agent Service gRPC
+ - "19908:19908" # Research Service gRPC
+ healthcheck:
+ test: ["CMD-SHELL", "nc -z localhost 8930 || exit 1"]
+ interval: 30s
+ timeout: 10s
+ start_period: 60s
+ retries: 3
+ volumes:
+ -
./dev-tools/deployment-scripts/airavata-server.properties:/opt/airavata/vault/airavata-server.properties:ro
+ -
./dev-tools/deployment-scripts/airavata-server.properties:/opt/airavata/apache-airavata-api-server/conf/airavata-server.properties:ro
+ - airavata-logs:/opt/airavata/logs
+ environment:
+ - JAVA_OPTS=-Xmx2g -Xms1g -Dthrift.server.max.message.size=1048576000
-Dthrift.server.max.frame.size=1048576000
-Dthrift.server.transport.max.message.size=1048576000
-Dthrift.maxMessageSize=1048576000 -Dthrift.maxFrameSize=1048576000
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
+ depends_on:
+ - mysql
+ - rabbitmq
+ - zookeeper
+ - kafka
+ networks:
+ - airavata-network
+
+volumes:
+ mysql-data:
+ rabbitmq-data:
+ zookeeper-data:
+ zookeeper-logs:
+ kafka-data:
+ airavata-logs:
+
+networks:
+ airavata-network:
+ driver: bridge
