This is an automated email from the ASF dual-hosted git repository. marat pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-karavan.git
commit e16bae115f4b1e3c5f4085554372170a4a2e97bb Author: Marat Gubaidullin <ma...@talismancloud.io> AuthorDate: Wed Sep 6 11:18:25 2023 -0400 Image builder for #817 --- karavan-web/docker/Dockerfile.devmode | 1 + .../camel/karavan/docker/DockerEventListener.java | 120 ++++----------------- .../camel/karavan/docker/DockerForKaravan.java | 9 +- .../apache/camel/karavan/docker/DockerService.java | 2 +- .../camel/karavan/docker/DockerServiceUtils.java | 2 +- .../apache/camel/karavan/service/EventService.java | 86 ++++----------- .../camel/karavan/service/ProjectService.java | 13 +-- .../org/apache/camel/karavan/shared/Constants.java | 1 + .../org/apache/camel/karavan/shared/EventType.java | 2 - .../snippets/camel-main-builder-script-docker.sh | 2 +- 10 files changed, 56 insertions(+), 182 deletions(-) diff --git a/karavan-web/docker/Dockerfile.devmode b/karavan-web/docker/Dockerfile.devmode index caa27a93..144ee1f6 100644 --- a/karavan-web/docker/Dockerfile.devmode +++ b/karavan-web/docker/Dockerfile.devmode @@ -22,6 +22,7 @@ RUN curl -Ls https://sh.jbang.dev | bash -s - app setup \ && mkdir -p $CODE_DIR \ && jbang trust add --quiet https://github.com/apache/camel \ && jbang app install camel@apache/camel + && jbang config set cache-evict never EXPOSE 8080 CMD jbang $JBANG_OPTIONS -Dcamel.jbang.version=$CAMEL_VERSION camel@apache/camel run --source-dir=$CODE_DIR --console --local-kamelet-dir=$KAMELETS_DIR --maven-settings=$MAVEN_SETTINGS diff --git a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerEventListener.java b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerEventListener.java index 86195c79..0af06f22 100644 --- a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerEventListener.java +++ b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerEventListener.java @@ -2,32 +2,34 @@ package org.apache.camel.karavan.docker; import com.github.dockerjava.api.async.ResultCallback; import com.github.dockerjava.api.model.Container; -import com.github.dockerjava.api.model.ContainerPort; import com.github.dockerjava.api.model.Event; import com.github.dockerjava.api.model.EventType; import io.vertx.core.eventbus.EventBus; -import io.vertx.core.json.JsonObject; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; import org.apache.camel.karavan.infinispan.InfinispanService; import org.apache.camel.karavan.infinispan.model.ContainerStatus; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.jboss.logging.Logger; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; import java.io.Closeable; import java.io.IOException; -import java.time.Instant; -import java.util.*; -import java.util.stream.Collectors; +import java.util.Objects; +import java.util.Optional; import static org.apache.camel.karavan.shared.Constants.*; -import static org.apache.camel.karavan.shared.EventType.*; @ApplicationScoped public class DockerEventListener implements ResultCallback<Event> { - @ConfigProperty(name = "karavan.environment") - String environment; + @ConfigProperty(name = "karavan.image-registry") + String registry; + @ConfigProperty(name = "karavan.image-group") + String group; + @ConfigProperty(name = "karavan.image-registry-username") + Optional<String> username; + @ConfigProperty(name = "karavan.image-registry-password") + Optional<String> password; @Inject DockerService dockerService; @@ -55,104 +57,22 @@ public class DockerEventListener implements ResultCallback<Event> { } } } catch (Exception exception) { - exception.printStackTrace(); + LOGGER.error(exception.getMessage()); } } - public void onContainerEvent(Event event, Container container) { - String status = event.getStatus(); + public void onContainerEvent(Event event, Container container) throws InterruptedException { if (infinispanService.isReady()) { - if (status.startsWith("health_status:")) { - if (inDevMode(container)) { - onDevModeHealthEvent(container, event); - } + if ("exited".equalsIgnoreCase(container.getState()) + && Objects.equals(container.getLabels().get(LABEL_TYPE), ContainerStatus.ContainerType.build.name())) { + String tag = container.getLabels().get(LABEL_TAG); + String projectId = container.getLabels().get(LABEL_PROJECT_ID); + String image = registry + "/" + group + "/" + projectId + ":" + tag; + dockerService.pullImage(image); } } } - public void onDeletedContainer(Container container) { - String name = container.getNames()[0].replace("/", ""); - infinispanService.deleteContainerStatus(name, environment, name); - if (inDevMode(container)) { - infinispanService.deleteCamelStatuses(name, environment); - } - } - - protected void onExistingContainer(Container container) { - if (infinispanService.isReady()) { - String name = container.getNames()[0].replace("/", ""); - List<Integer> ports = Arrays.stream(container.getPorts()).map(ContainerPort::getPrivatePort).filter(Objects::nonNull).collect(Collectors.toList()); - List<ContainerStatus.Command> commands = getContainerCommand(container.getState()); - ContainerStatus.ContainerType type = getContainerType(container.getLabels()); - String created = Instant.ofEpochSecond(container.getCreated()).toString(); - String projectId = container.getLabels().getOrDefault(LABEL_PROJECT_ID, name); - ContainerStatus ci = infinispanService.getContainerStatus(projectId, environment, name); - if (ci == null) { - ci = ContainerStatus.createWithId(projectId, name, environment, container.getId(), container.getImage(), ports, type, commands, container.getState(), created); - } else { - ci.setContainerId(container.getId()); - ci.setPorts(ports); - ci.setType(type); - ci.setCommands(commands); - ci.setCreated(created); - ci.setState(container.getState()); - ci.setImage(container.getImage()); - } - infinispanService.saveContainerStatus(ci); - } - } - - public void onDevModeHealthEvent(Container container, Event event) { - String status = event.getStatus(); - String health = status.replace("health_status: ", ""); - LOGGER.infof("Container %s health status: %s", container.getNames()[0], health); - Map<String, Object> message = Map.of( - LABEL_PROJECT_ID, container.getLabels().get(LABEL_PROJECT_ID), - RELOAD_TRY_COUNT, 1 - ); - eventBus.publish(DEVMODE_CONTAINER_READY, JsonObject.mapFrom(message)); - } - - private boolean inDevMode(Container container) { - return Objects.equals(getContainerType(container.getLabels()), ContainerStatus.ContainerType.devmode); - } - - private ContainerStatus.ContainerType getContainerType(Map<String, String> labels) { - String type = labels.get(LABEL_TYPE); - if (Objects.equals(type, ContainerStatus.ContainerType.devmode.name())) { - return ContainerStatus.ContainerType.devmode; - } else if (Objects.equals(type, ContainerStatus.ContainerType.devservice.name())) { - return ContainerStatus.ContainerType.devservice; - } else if (Objects.equals(type, ContainerStatus.ContainerType.project.name())) { - return ContainerStatus.ContainerType.project; - } else if (Objects.equals(type, ContainerStatus.ContainerType.internal.name())) { - return ContainerStatus.ContainerType.internal; - } - return ContainerStatus.ContainerType.unknown; - } - - private List<ContainerStatus.Command> getContainerCommand(String state) { - List<ContainerStatus.Command> result = new ArrayList<>(); - if (Objects.equals(state, ContainerStatus.State.created.name())) { - result.add(ContainerStatus.Command.run); - result.add(ContainerStatus.Command.delete); - } else if (Objects.equals(state, ContainerStatus.State.exited.name())) { - result.add(ContainerStatus.Command.run); - result.add(ContainerStatus.Command.delete); - } else if (Objects.equals(state, ContainerStatus.State.running.name())) { - result.add(ContainerStatus.Command.pause); - result.add(ContainerStatus.Command.stop); - result.add(ContainerStatus.Command.delete); - } else if (Objects.equals(state, ContainerStatus.State.paused.name())) { - result.add(ContainerStatus.Command.run); - result.add(ContainerStatus.Command.stop); - result.add(ContainerStatus.Command.delete); - } else if (Objects.equals(state, ContainerStatus.State.dead.name())) { - result.add(ContainerStatus.Command.delete); - } - return result; - } - @Override public void onError(Throwable throwable) { LOGGER.error(throwable.getMessage()); diff --git a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerForKaravan.java b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerForKaravan.java index 474265d2..41d15c7f 100644 --- a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerForKaravan.java +++ b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerForKaravan.java @@ -69,19 +69,20 @@ public class DockerForKaravan { } - public void runBuildProject(String projectId, String script, Map<String, String> files, List<String> env) throws Exception { - Container c = createBuildContainer(projectId, env); + public void runBuildProject(String projectId, String script, Map<String, String> files, List<String> env, String tag) throws Exception { + dockerService.deleteContainer(projectId + BUILDER_SUFFIX); + Container c = createBuildContainer(projectId, env, tag); dockerService.copyFiles(c.getId(), "/code", files); dockerService.copyExecFile(c.getId(), "/karavan", "build.sh", script); dockerService.runContainer(projectId); } - protected Container createBuildContainer(String projectId, List<String> env) throws InterruptedException { + protected Container createBuildContainer(String projectId, List<String> env, String tag) throws InterruptedException { LOGGER.infof("Starting Build Container for %s ", projectId); return dockerService.createContainer(projectId + BUILDER_SUFFIX, devmodeImage, env, Map.of(), new HealthCheck(), - Map.of(LABEL_TYPE, ContainerStatus.ContainerType.build.name(), LABEL_PROJECT_ID, projectId), + Map.of(LABEL_TYPE, ContainerStatus.ContainerType.build.name(), LABEL_PROJECT_ID, projectId, LABEL_TAG, tag), Map.of(), "/karavan/build.sh"); } diff --git a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerService.java b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerService.java index fcceccd2..fcbee5a4 100644 --- a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerService.java +++ b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerService.java @@ -90,7 +90,7 @@ public class DockerService extends DockerServiceUtils { getDockerClient().listContainersCmd().withShowAll(true).exec().forEach(container -> { ContainerStatus containerStatus = getContainerStatus(container, environment); Statistics stats = getContainerStats(container.getId()); - updateStatistics(containerStatus, container, stats); + updateStatistics(containerStatus, stats); result.add(containerStatus); }); return result; diff --git a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerServiceUtils.java b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerServiceUtils.java index 91ae2cf9..ebbd8403 100644 --- a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerServiceUtils.java +++ b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/docker/DockerServiceUtils.java @@ -53,7 +53,7 @@ public class DockerServiceUtils { return ContainerStatus.createWithId(projectId, name, environment, container.getId(), container.getImage(), ports, type, commands, container.getState(), created); } - protected void updateStatistics(ContainerStatus containerStatus, Container container, Statistics stats) { + protected void updateStatistics(ContainerStatus containerStatus, Statistics stats) { if (stats != null && stats.getMemoryStats() != null) { String memoryUsage = formatMemory(stats.getMemoryStats().getUsage()); String memoryLimit = formatMemory(stats.getMemoryStats().getLimit()); diff --git a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/service/EventService.java b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/service/EventService.java index c74dde56..b1a9ee10 100644 --- a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/service/EventService.java +++ b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/service/EventService.java @@ -1,79 +1,26 @@ package org.apache.camel.karavan.service; import io.quarkus.vertx.ConsumeEvent; -import io.vertx.core.eventbus.EventBus; import io.vertx.core.json.JsonObject; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.apache.camel.karavan.infinispan.InfinispanService; -import org.apache.camel.karavan.infinispan.model.CamelStatus; import org.apache.camel.karavan.infinispan.model.ContainerStatus; -import org.eclipse.microprofile.config.inject.ConfigProperty; import org.jboss.logging.Logger; import java.time.Instant; import java.util.Objects; import static org.apache.camel.karavan.shared.EventType.CONTAINER_STATUS; -import static org.apache.camel.karavan.shared.EventType.DEVMODE_CONTAINER_READY; @ApplicationScoped public class EventService { private static final Logger LOGGER = Logger.getLogger(EventService.class.getName()); - @ConfigProperty(name = "karavan.environment") - String environment; - @Inject InfinispanService infinispanService; - @Inject - CamelService camelService; - - @Inject - EventBus eventBus; - - @ConsumeEvent(value = DEVMODE_CONTAINER_READY, blocking = true, ordered = true) - void receiveCommand(JsonObject json) { -// String projectId = json.getString(LABEL_PROJECT_ID); -// Integer reloadCount = json.getInteger(RELOAD_TRY_COUNT); -// LOGGER.info("DEVMODE_CONTAINER_READY " + projectId + " : " + reloadCount); -// ContainerStatus status = infinispanService.getContainerStatus(projectId, environment, projectId); -// CamelStatus cs = infinispanService.getCamelStatus(projectId, environment, CamelStatus.Name.context.name()); -// if (status != null -// && !Objects.equals(status.getCodeLoaded(), Boolean.TRUE) -// && status.getContainerId() != null -// && status.getState().equals(ContainerStatus.State.running.name()) -// && camelIsStarted(cs)) { -// LOGGER.info("CAMEL STARTED -> SEND RELOAD"); -// if (ConfigService.inKubernetes()) { -// camelService.reloadProjectCode(projectId); -// } else { -// infinispanService.sendCodeReloadCommand(projectId); -// } -// } else if (reloadCount < 30) { -// LOGGER.info("CAMEL NOT STARTED -> SEND DEVMODE_CONTAINER_READY"); -// // retry again -// Map<String, Object> message = Map.of( -// LABEL_PROJECT_ID, projectId, -// RELOAD_TRY_COUNT, ++reloadCount -// ); -// eventBus.publish(DEVMODE_DELAY_MESSAGE, JsonObject.mapFrom(message)); -// } - } - - private boolean camelIsStarted(CamelStatus camelStatus) { - try { - String status = camelStatus.getStatus(); - JsonObject obj = new JsonObject(status); - return Objects.equals("Started", obj.getJsonObject("context").getString("state")); - } catch (Exception e) { - return false; - } - } - - @ConsumeEvent(value = CONTAINER_STATUS, blocking = true, ordered = true) public void saveContainerStatus(JsonObject data) { @@ -83,23 +30,28 @@ public class EventService { if (oldStatus == null) { infinispanService.saveContainerStatus(newStatus); } else if (Objects.equals(oldStatus.getInTransit(), Boolean.FALSE)) { - if ("exited".equalsIgnoreCase(newStatus.getState()) && oldStatus.getFinished() == null) { - newStatus.setFinished(Instant.now().toString()); - } - if (newStatus.getCpuInfo() == null) { - newStatus.setCpuInfo(oldStatus.getCpuInfo()); - newStatus.setMemoryInfo(oldStatus.getMemoryInfo()); - } - infinispanService.saveContainerStatus(newStatus); + saveContainerStatus(newStatus, oldStatus); } else if (Objects.equals(oldStatus.getInTransit(), Boolean.TRUE)) { - if (!Objects.equals(oldStatus.getState(), newStatus.getState())) { - if (newStatus.getCpuInfo() == null) { - newStatus.setCpuInfo(oldStatus.getCpuInfo()); - newStatus.setMemoryInfo(oldStatus.getMemoryInfo()); - } - infinispanService.saveContainerStatus(newStatus); + if (!Objects.equals(oldStatus.getState(), newStatus.getState()) || newStatus.getCpuInfo().isEmpty()) { + saveContainerStatus(newStatus, oldStatus); } } } } + + private void saveContainerStatus(ContainerStatus newStatus, ContainerStatus oldStatus) { + if (newStatus.getProjectId().equals("demo")) { + System.out.println(("oldStatus.getFinished = " + Objects.isNull(oldStatus.getFinished()))); + } + if ("exited".equalsIgnoreCase(newStatus.getState()) && Objects.isNull(oldStatus.getFinished())) { + newStatus.setFinished(Instant.now().toString()); + } else if ("exited".equalsIgnoreCase(newStatus.getState()) && Objects.nonNull(oldStatus.getFinished())) { + return; + } + if (newStatus.getCpuInfo() == null || newStatus.getCpuInfo().isEmpty()) { + newStatus.setCpuInfo(oldStatus.getCpuInfo()); + newStatus.setMemoryInfo(oldStatus.getMemoryInfo()); + } + infinispanService.saveContainerStatus(newStatus); + } } \ No newline at end of file diff --git a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/service/ProjectService.java b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/service/ProjectService.java index 84f8ee0d..d3489247 100644 --- a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/service/ProjectService.java +++ b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/service/ProjectService.java @@ -141,14 +141,14 @@ public class ProjectService implements HealthCheck { String templateName = project.getRuntime() + "-builder-script-docker.sh"; String script = codeService.getTemplateText(templateName); - List<String> env = getEnvForBuild(project); - - dockerForKaravan.runBuildProject(project.getProjectId(), script, files, env); + String tag = Instant.now().toString().substring(0, 19).replace(":", "-"); + List<String> env = getEnvForBuild(project, tag); + dockerForKaravan.runBuildProject(project.getProjectId(), script, files, env, tag); return project.getProjectId(); } } - private List<String> getEnvForBuild(Project project) { + private List<String> getEnvForBuild(Project project, String tag) { GitConfig gitConfig = gitService.getGitConfig(); List<String> env = List.of( "GIT_REPOSITORY=" + (installGitea ? gitConfig.getUri().replace("localhost", "gitea") : gitConfig.getUri()), @@ -159,7 +159,9 @@ public class ProjectService implements HealthCheck { "IMAGE_REGISTRY=" + (installRegistry ? "registry:5000" : registry), "IMAGE_REGISTRY_USERNAME=" + username, "IMAGE_REGISTRY_PASSWORD=" + password, - "IMAGE_GROUP=" + group + "IMAGE_GROUP=" + group, + "TAG=" + tag + ); return env; } @@ -274,7 +276,6 @@ public class ProjectService implements HealthCheck { p.setLastCommit(commitId); p.setLastCommitTimestamp(lastUpdate); infinispanService.saveProject(p); - infinispanService.saveCommit(commitId, commit.getCommitTime()); return p; } diff --git a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/shared/Constants.java b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/shared/Constants.java index 3b77038f..1e38924c 100644 --- a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/shared/Constants.java +++ b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/shared/Constants.java @@ -22,6 +22,7 @@ public class Constants { public static final String LABEL_TYPE = "org.apache.camel.karavan/type"; public static final String LABEL_PROJECT_ID = "org.apache.camel.karavan/projectId"; + public static final String LABEL_TAG = "org.apache.camel.karavan/tag"; public static final String RELOAD_TRY_COUNT = "reloadTryCount"; diff --git a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/shared/EventType.java b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/shared/EventType.java index 480185ab..add236cd 100644 --- a/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/shared/EventType.java +++ b/karavan-web/karavan-app/src/main/java/org/apache/camel/karavan/shared/EventType.java @@ -19,6 +19,4 @@ package org.apache.camel.karavan.shared; public class EventType { public static final String CONTAINER_STATUS = "CONTAINER_STATUS"; - public static final String DEVMODE_CONTAINER_READY = "DEVMODE_CONTAINER_READY"; - } diff --git a/karavan-web/karavan-app/src/main/resources/snippets/camel-main-builder-script-docker.sh b/karavan-web/karavan-app/src/main/resources/snippets/camel-main-builder-script-docker.sh index 62b72584..207520cc 100644 --- a/karavan-web/karavan-app/src/main/resources/snippets/camel-main-builder-script-docker.sh +++ b/karavan-web/karavan-app/src/main/resources/snippets/camel-main-builder-script-docker.sh @@ -24,7 +24,7 @@ cd ${CHECKOUT_DIR}/${PROJECT_ID} jbang -Dcamel.jbang.version=${CAMEL_VERSION} camel@apache/camel export --local-kamelet-dir=${KAMELETS_DIR} export LAST_COMMIT=$(git rev-parse --short HEAD) -export DATE=$(date '+%Y%m%d%H%M%S') +export DATE=${TAG} mvn package jib:build \ -Djib.allowInsecureRegistries=true \