Guruprasad has proposed merging ~lgp171188/launchpad:charm-launchpad-codehosting into launchpad:master.
Commit message: Add a launchpad-codehosting charm Requested reviews: Launchpad code reviewers (launchpad-reviewers) For more details, see: https://code.launchpad.net/~lgp171188/launchpad/+git/launchpad/+merge/452140 -- Your team Launchpad code reviewers is requested to review the proposed merge of ~lgp171188/launchpad:charm-launchpad-codehosting into launchpad:master.
diff --git a/charm/launchpad-codehosting/actions.yaml b/charm/launchpad-codehosting/actions.yaml new file mode 100644 index 0000000..86aae0a --- /dev/null +++ b/charm/launchpad-codehosting/actions.yaml @@ -0,0 +1,11 @@ +start-services: + description: | + Start the launchpad-bzr-sftp service. Usually run after + maintenance. +stop-services: + description: | + Stop the launchpad-bzr-sftp service. Usually run in preparation + for maintenance. (Note that this does not stop services in a way that + will persist across a reboot. It also doesn't disable cron jobs, since + those are handled by the cron-control mechanism instead; see + lp.services.scripts.base.cronscript_enabled.) diff --git a/charm/launchpad-codehosting/actions/actions.py b/charm/launchpad-codehosting/actions/actions.py new file mode 100644 index 0000000..cbac43e --- /dev/null +++ b/charm/launchpad-codehosting/actions/actions.py @@ -0,0 +1,53 @@ +#! /usr/bin/python3 +# Copyright 2023 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +import subprocess +import sys +import traceback +from pathlib import Path + +sys.path.append("lib") + +from charms.layer import basic # noqa: E402 + +basic.bootstrap_charm_deps() +basic.init_config_states() + +from charmhelpers.core import hookenv # noqa: E402 + + +def start_services(): + service = "launchpad-bzr-sftp.service" + hookenv.log(f"Starting {service}.") + subprocess.run(["systemctl", "start", service], check=True) + hookenv.action_set({"result": "Services started"}) + + +def stop_services(): + service = "launchpad-bzr-sftp.service" + hookenv.log(f"Stopping {service}.") + subprocess.run(["systemctl", "stop", service], check=True) + hookenv.action_set({"result": "Services stopped"}) + + +def main(argv): + action = Path(argv[0]).name + try: + if action == "start-services": + start_services() + elif action == "stop-services": + stop_services() + else: + hookenv.action_fail(f"Action {action} not implemented.") + except Exception: + hookenv.action_fail("Unhandled exception") + tb = traceback.format_exc() + hookenv.action_set(dict(traceback=tb)) + hookenv.log(f"Unhandled exception in action {action}:") + for line in tb.splitlines(): + hookenv.log(line) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/charm/launchpad-codehosting/actions/start-services b/charm/launchpad-codehosting/actions/start-services new file mode 120000 index 0000000..405a394 --- /dev/null +++ b/charm/launchpad-codehosting/actions/start-services @@ -0,0 +1 @@ +actions.py \ No newline at end of file diff --git a/charm/launchpad-codehosting/actions/stop-services b/charm/launchpad-codehosting/actions/stop-services new file mode 120000 index 0000000..405a394 --- /dev/null +++ b/charm/launchpad-codehosting/actions/stop-services @@ -0,0 +1 @@ +actions.py \ No newline at end of file diff --git a/charm/launchpad-codehosting/charmcraft.yaml b/charm/launchpad-codehosting/charmcraft.yaml new file mode 100644 index 0000000..302c2f4 --- /dev/null +++ b/charm/launchpad-codehosting/charmcraft.yaml @@ -0,0 +1,76 @@ +type: charm +bases: + - build-on: + - name: ubuntu + channel: "20.04" + architectures: [amd64] + run-on: + - name: ubuntu + channel: "20.04" + architectures: [amd64] +parts: + charm-wheels: + source: https://git.launchpad.net/~ubuntuone-hackers/ols-charm-deps/+git/wheels + source-commit: "42c89d9c66dbe137139b047fd54aed49b66d1a5e" + source-submodules: [] + source-type: git + plugin: dump + organize: + "*": charm-wheels/ + prime: + - "-charm-wheels" + ols-layers: + source: https://git.launchpad.net/ols-charm-deps + source-commit: "9c59a9804f1f40e2a74be7dac9bf18a655a7864f" + source-submodules: [] + source-type: git + plugin: dump + organize: + "*": layers/ + stage: + - layers + prime: + - "-layers" + launchpad-layers: + after: + - ols-layers + source: https://git.launchpad.net/launchpad-layers + source-commit: "58edb3e5a88794c3baa2274a94e21d3a298a6c79" + source-submodules: [] + source-type: git + plugin: dump + organize: + launchpad-base: layers/layer/launchpad-base + launchpad-db: layers/layer/launchpad-db + launchpad-payload: layers/layer/launchpad-payload + stage: + - layers + prime: + - "-layers" + interface-apache-website: + source: https://github.com/juju-solutions/interface-apache-website + source-commit: "2f736ebcc90d19ac142a2d898a2ec7e1aafaa13f" + source-submodules: [] + source-type: git + plugin: dump + organize: + "*": layers/interface/apache-website/ + stage: + - layers + prime: + - "-layers" + charm: + after: + - charm-wheels + - launchpad-layers + - interface-apache-website + source: . + plugin: reactive + build-snaps: [charm] + build-packages: [libpq-dev, python3-dev] + build-environment: + - CHARM_LAYERS_DIR: $CRAFT_STAGE/layers/layer + - CHARM_INTERFACES_DIR: $CRAFT_STAGE/layers/interface + - PIP_NO_INDEX: "true" + - PIP_FIND_LINKS: $CRAFT_STAGE/charm-wheels + reactive-charm-build-arguments: [--binary-wheels-from-source] diff --git a/charm/launchpad-codehosting/config.yaml b/charm/launchpad-codehosting/config.yaml new file mode 100644 index 0000000..68f4a1b --- /dev/null +++ b/charm/launchpad-codehosting/config.yaml @@ -0,0 +1,46 @@ +options: + active: + type: boolean + description: If true, enable jobs that may change the database. + default: true + blocklisted_hostnames: + type: string + description: Comma-separated list of hostnames to blocklist. + default: "localhost,127.0.0.1" + codebrowse_internal_endpoint: + type: string + description: | + The internal-only endpoint at which the codebrowse service is + accessible. + codehosting_private_ssh_key: + type: string + description: > + Base64-encoded private SSH RSA host key to be used by the codehosting + service. Existing key pair, if any, will be deleted if this is unset. + default: "" + codehosting_public_ssh_key: + type: string + description: > + Base64-encoded public SSH RSA host key to be used by the codehosting + service. Existing key pair, if any, will be deleted if this is unset. + default: "" + domain_bzr_internal: + type: string + description: | + The internal-only domain name to expose the bazaar get branch by ID + service on. + port_bzr_sftp_base: + type: int + description: Base port number for the bzr-sftp service. + default: 2224 + port_web_status_base: + type: int + description: Base port for the web status service. + default: 8024 + workers: + type: int + description: > + Number of bzr-sftp worker processes. If set, each worker will listen + on consecutive ports starting from each of `port_bzr_sftp_base` and + `port_web_status`, so make sure there is enough space between those. + default: 1 diff --git a/charm/launchpad-codehosting/layer.yaml b/charm/launchpad-codehosting/layer.yaml new file mode 100644 index 0000000..408f80b --- /dev/null +++ b/charm/launchpad-codehosting/layer.yaml @@ -0,0 +1,10 @@ +includes: + - layer:launchpad-db + - interface:apache-website +options: + ols-pg: + databases: + db: + name: launchpad_dev + roles: [] +repo: https://git.launchpad.net/launchpad diff --git a/charm/launchpad-codehosting/metadata.yaml b/charm/launchpad-codehosting/metadata.yaml new file mode 100644 index 0000000..43c7884 --- /dev/null +++ b/charm/launchpad-codehosting/metadata.yaml @@ -0,0 +1,19 @@ +name: launchpad-codehosting +display-name: launchpad-codehosting +summary: Launchpad bazaar codehosting service +maintainer: Launchpad Developers <launchpad-...@lists.launchpad.net> +description: | + Launchpad is an open source suite of tools that help people and teams + to work together on software projects. + + This charm runs the Launchpad bazaar code hosting service. +tags: + # https://juju.is/docs/charm-metadata#heading--charm-store-fields + - network +series: + - focal +subordinate: true +requires: + apache-website: + interface: apache-website + scope: container diff --git a/charm/launchpad-codehosting/reactive/launchpad-codehosting.py b/charm/launchpad-codehosting/reactive/launchpad-codehosting.py new file mode 100644 index 0000000..ccfe765 --- /dev/null +++ b/charm/launchpad-codehosting/reactive/launchpad-codehosting.py @@ -0,0 +1,313 @@ +# Copyright 2023 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +import base64 +import os +import subprocess + +from charmhelpers.core import hookenv, host, templating +from charms.launchpad.base import configure_email, get_service_config +from charms.launchpad.db import lazr_config_files +from charms.launchpad.payload import ( + config_file_path, + configure_cron, + configure_lazr, +) +from charms.reactive import ( + clear_flag, + endpoint_from_flag, + helpers, + remove_state, + set_flag, + set_state, + when, + when_not, + when_not_all, +) +from ols import base + + +def base64_decode(value): + return base64.b64decode(value.encode("ASCII")) + + +def configure_logrotate(config): + hookenv.log("Writing logrotate configuration.") + templating.render( + "logrotate.conf.j2", + "/etc/logrotate.d/launchpad", + config, + perms=0o644, + ) + + +def configure_systemd(config): + hookenv.log("Writing launchpad-bzr-sftp systemd service.") + templating.render( + "launchpad-bzr-sftp.service.j2", + "/lib/systemd/system/launchpad-bzr-sftp.service", + config, + ) + templating.render( + "launchpad-bzr-sftp@.service.j2", + "/lib/systemd/system/launchpad-bzr-sftp@.service", + config, + ) + templating.render( + "launchpad-bzr-sftp-generator.j2", + "/lib/systemd/system-generators/launchpad-bzr-sftp-generator", + config, + perms=0o555, + ) + subprocess.run(["systemctl", "daemon-reload"]) + + +def config_files(): + files = [] + files.extend(lazr_config_files()) + config = get_service_config() + for i in range(config["workers"]): + files.append( + config_file_path( + f"launchpad-codehosting{i + 1}/launchpad-lazr.conf" + ) + ) + files.append(config_file_path("launchpad-codehosting-secrets-lazr.conf")) + return files + + +def configure_scripts(config): + hookenv.log(f"Creating {config['scripts_dir']}, if doesn't exist already.") + host.mkdir( + config["scripts_dir"], + owner=config["user"], + group=config["user"], + perms=0o755, + force=True, + ) + hookenv.log("Writing the cleanlogs script.") + templating.render( + "cleanlogs.j2", + f"{config['scripts_dir']}/cleanlogs", + config, + perms=0o755, + ) + host.mkdir( + f"{config['logs_dir']}/sftp-logs", + owner=config["user"], + group=config["user"], + perms=0o755, + force=True, + ) + hookenv.log("Writing the branch rewrite wrapper script.") + templating.render( + "rewrite_wrapper.sh.j2", + f"{config['scripts_dir']}/rewrite_wrapper.sh", + config, + owner=config["user"], + group=config["user"], + perms=0o755, + ) + + +def configure_ssh_keys(config): + host_key_pair_path = f"/srv/{config['domain_bzr']}/keys" + hookenv.log(f"Creating {host_key_pair_path} to store the SSH host keys.") + user = config["user"] + host.mkdir( + host_key_pair_path, + owner=user, + group=user, + perms=0o755, + force=True, + ) + ssh_private_key_file = f"{host_key_pair_path}/ssh_host_key_rsa" + ssh_public_key_file = f"{ssh_private_key_file}.pub" + if ( + config["codehosting_private_ssh_key"] + and config["codehosting_public_ssh_key"] + ): + hookenv.log("Writing the SSH host key pair.") + host.write_file( + ssh_private_key_file, + base64_decode(config["codehosting_private_ssh_key"]), + owner=user, + group=user, + perms=0o600, + ) + host.write_file( + ssh_public_key_file, + base64_decode(config["codehosting_public_ssh_key"]), + owner=user, + group=user, + perms=0o644, + ) + else: + hookenv.log( + "SSH key pair not configured. Deleting existing keys, if present." + ) + for path in (ssh_private_key_file, ssh_public_key_file): + if os.path.exists(path): + os.unlink(path) + + +def configure_codehosting_lazr_config(config): + hookenv.log("Writing lazr configuration.") + for i in range(config["workers"]): + config["service_access_log_file"] = f"codehosting-{i + 1}-access.log" + config["service_sftp_port"] = config["port_bzr_sftp_base"] + i + config["service_web_status_port"] = config["port_web_status_base"] + i + config["service_oops_prefix"] = f"{config['oops_prefix']}{i + 1}" + configure_lazr( + config, + "launchpad-codehosting-lazr.conf.j2", + f"launchpad-codehosting{i + 1}/launchpad-lazr.conf", + ) + configure_lazr( + config, + "launchpad-codehosting-secrets-lazr.conf.j2", + "launchpad-codehosting-secrets-lazr.conf", + secret=True, + ) + + +@when("launchpad.db.configured") +@when_not("service.configured") +def configure(): + config = get_service_config() + configure_codehosting_lazr_config(config) + configure_email(config, "launchpad-codehosting") + configure_logrotate(config) + configure_cron(config, "crontab.j2") + configure_scripts(config) + configure_ssh_keys(config) + configure_systemd(config) + if config["active"]: + if helpers.any_file_changed( + [ + base.version_info_path(), + "/lib/systemd/system/launchpad-bzr-sftp.service", + "/lib/systemd/system/launchpad-bzr-sftp@.service", + "/lib/systemd/system-generators/launchpad-bzr-sftp-generator", + ] + + config_files() + ): + hookenv.log( + "Config files changed; restarting" + " the launchpad-bzr-sftp service." + ) + for i in range(config["workers"]): + host.service_restart(f"launchpad-bzr-sftp@{i + 1}") + else: + hookenv.log("Not restarting since no config files were changed.") + host.service_resume("launchpad-bzr-sftp.service") + + set_state("service.configured") + + +def get_vhost_config(config): + hookenv.log("Rendering the virtual hosts configuration.") + return "\n".join( + [ + templating.render("vhosts/common.conf", None, config), + templating.render("vhosts/bazaar_http.conf.j2", None, config), + templating.render("vhosts/bazaar_https.conf.j2", None, config), + templating.render( + "vhosts/bazaar_internal_branch_by_id.conf.j2", None, config + ), + ] + ) + + +def configure_document_root(config): + hookenv.log("Configuring the document root.") + document_root_base_dir = f"/srv/{config['domain_bzr']}" + user = config["user"] + host.mkdir( + document_root_base_dir, + owner=user, + group="root", + perms=0o755, + force=True, + ) + config["bzr_repositories_root"] = f"{document_root_base_dir}/mirrors" + host.mkdir( + config["bzr_repositories_root"], + owner=user, + group=user, + perms=0o755, + force=True, + ) + + static_files_root = f"{document_root_base_dir}/www" + config["static_files_root"] = static_files_root + host.mkdir( + static_files_root, + owner=user, + group=user, + perms=0o755, + force=True, + ) + templating.render( + "robots.txt", + f"{static_files_root}/robots.txt", + config, + owner=user, + group=user, + perms=0o644, + ) + code_dir = config["code_dir"] + host.symlink( + f"{code_dir}/lib/canonical/launchpad/offline-unplanned.html", + f"{static_files_root}/offline.html", + ) + site_packages_dir = ( + subprocess.check_output( + [ + f"{code_dir}/env/bin/python", + "-c", + "import sysconfig; print(sysconfig.get_path('purelib'))", + ] + ) + .strip() + .decode("utf-8") + ) + assert site_packages_dir.startswith(f"{code_dir}/env") + host.symlink( + f"{site_packages_dir}/loggerhead/static", f"{static_files_root}/static" + ) + + +@when( + "service.configured", + "config.set.domain_bzr", + "config.set.domain_bzr_internal", + "apache-website.available", +) +@when_not("service.apache-website.configured") +def configure_apache_website(): + apache_website = endpoint_from_flag("apache-website.available") + config = get_service_config() + configure_document_root(config) + apache_website.set_remote( + domain=config["domain_bzr"], + enabled="true", + ports="80 8080 8081", + site_config=get_vhost_config(config), + site_modules="headers proxy proxy_http rewrite", + ) + hookenv.status_set("active", "Ready") + set_flag("service.apache-website.configured") + + +@when("service.apache-website.configured") +@when_not_all("service.configured", "apache-website.available") +def apache_deconfigured(): + hookenv.status_set("blocked", "Website not yet configured") + clear_flag("service.apache-website.configured") + + +@when("service.configured") +@when_not("launchpad.db.configured") +def deconfigure(): + remove_state("service.configured") diff --git a/charm/launchpad-codehosting/templates/cleanlogs.j2 b/charm/launchpad-codehosting/templates/cleanlogs.j2 new file mode 100644 index 0000000..b2cf752 --- /dev/null +++ b/charm/launchpad-codehosting/templates/cleanlogs.j2 @@ -0,0 +1,17 @@ +#!/bin/bash + +ROOTDIR={{ logs_dir }}/sftp-logs +DEST=$ROOTDIR/archives/$(date +%F) + +mkdir -p $DEST +find -L $ROOTDIR -maxdepth 1 -type f -name 'bzr-sftp?.log.*' -exec mv {} $DEST/ \; + +cd $DEST +for i in $(ls) +do + bzip2 $i +done + +# Remove older than 90 days +find -L $ROOTDIR/archives -maxdepth 1 -type d -mtime +90 -print0 | xargs -0r rm -r + diff --git a/charm/launchpad-codehosting/templates/crontab.j2 b/charm/launchpad-codehosting/templates/crontab.j2 new file mode 100644 index 0000000..b6ae48d --- /dev/null +++ b/charm/launchpad-codehosting/templates/crontab.j2 @@ -0,0 +1,29 @@ +TZ=UTC +MAILFROM={{ bounce_address }} +MAILTO={{ cron_mailto }} +LPCONFIG=launchpad-codehosting + +{%- if active %} + +* * * * * {% if http_proxy %}http_proxy={{ http_proxy }} https_proxy={{ http_proxy }}{% endif %}{{ code_dir }}/cronscripts/supermirror-pull.py -q --log-file=INFO:{{ logs_dir }}/puller.log + +# remove from disk, deleted branches +10 0 * * * {{ code_dir }}/cronscripts/process-job-source.py IReclaimBranchSpaceJobSource -q --log-file=DEBUG:{{ logs_dir }}/process-job-source.IReclaimBranchSpaceJobSource.log + +# Archive supermirror SFTP log files per https://portal.admin.canonical.com/C26974 +5 0 * * * {{ scripts_dir }}/cleanlogs + +# Translations to branch script https://portal.admin.canonical.com/C35040 +30 04 * * * {{ code_dir }}/cronscripts/translations-export-to-branch.py -q --log-file=DEBUG:{{ logs_dir }}/translations-export-to-branch.log + +# Upgrade branches script +*/10 * * * * {{ code_dir }}/cronscripts/process-job-source.py IBranchUpgradeJobSource -q --log-file=DEBUG:{{ logs_dir }}/process-job-source.IBranchUpgradeJobSource.log + +# cleanup old crud in /tmp cf. https://bugs.launchpad.net/launchpad/+bug/979511 +6 4 * * * find /tmp -maxdepth 1 -name 'bzr-index-*' -type f -mtime +15 -delete + +{%- endif %} + +# OOPS amqp +*/15 * * * * {{ code_dir }}/bin/datedir2amqp --exchange oopses --host {{ rabbitmq_host }} --username {{ rabbitmq_username }} --password {{ rabbitmq_password }} --vhost {{ rabbitmq_vhost }} --repo {{ oopses_dir }} --key "" + diff --git a/charm/launchpad-codehosting/templates/launchpad-bzr-sftp-generator.j2 b/charm/launchpad-codehosting/templates/launchpad-bzr-sftp-generator.j2 new file mode 100644 index 0000000..af3670b --- /dev/null +++ b/charm/launchpad-codehosting/templates/launchpad-bzr-sftp-generator.j2 @@ -0,0 +1,18 @@ +#! /bin/sh +# Copyright 2023 Canonical Ltd. This software is licensed under the +# GNU Affero General Public License version 3 (see the file LICENSE). + +# Part of the launchpad-codehosting Juju charm. + +set -e + +wantdir="$1/launchpad-bzr-sftp.service.wants" +template=/lib/systemd/system/launchpad-bzr-sftp@.service + +# Generate systemd unit dependency symlinks for all configured +# launchpad-bzr-sftp instances. +mkdir -p "$wantdir" +for i in $(seq {{ workers }}); do + ln -s "$template" "$wantdir/launchpad-bzr-sftp@$i.service" +done + diff --git a/charm/launchpad-codehosting/templates/launchpad-bzr-sftp.service.j2 b/charm/launchpad-codehosting/templates/launchpad-bzr-sftp.service.j2 new file mode 100644 index 0000000..36dec59 --- /dev/null +++ b/charm/launchpad-codehosting/templates/launchpad-bzr-sftp.service.j2 @@ -0,0 +1,18 @@ +# This service is really a systemd target, but we use a service since +# targets cannot be reloaded. Starting, stopping, or reloading this service +# causes all the individual instances (defined in +# launchpad-bzr-sftp@.service) to be started, stopped, or reloaded +# respectively. + +[Unit] +Description=Launchpad bzr sftp + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/true +ExecReload=/bin/true + +[Install] +WantedBy=multi-user.target + diff --git a/charm/launchpad-codehosting/templates/launchpad-bzr-sftp@.service.j2 b/charm/launchpad-codehosting/templates/launchpad-bzr-sftp@.service.j2 new file mode 100644 index 0000000..d3c9b57 --- /dev/null +++ b/charm/launchpad-codehosting/templates/launchpad-bzr-sftp@.service.j2 @@ -0,0 +1,24 @@ +[Unit] +Description=Launchpad bzr sftp service (%i) +PartOf=launchpad-bzr-sftp.service +Before=launchpad-bzr-sftp.service +ReloadPropagatedFrom=launchpad-bzr-sftp.service +After=network.target +ConditionPathExists=!{{ code_dir }}/maintenance.txt + +[Service] +User=launchpad +Group=launchpad +WorkingDirectory={{ code_dir }} +# https://portal.admin.canonical.com/C44221 +Environment=LPCONFIG=launchpad-codehosting%i +SyslogIdentifier=bzr-sftp +ExecStart={{ code_dir }}/bin/twistd --python daemons/sftp.tac --pidfile {{ var_dir }}/bzr-sftp%i.pid --prefix bzr-sftp --logfile {{ logs_dir }}/sftp-logs/bzr-sftp%i.log --nodaemon +ExecReload=/bin/kill -USR1 $MAINPID +KillMode=mixed +Restart=on-failure +PrivateTmp=true + +[Install] +WantedBy=multi-user.target + diff --git a/charm/launchpad-codehosting/templates/launchpad-codehosting-lazr.conf.j2 b/charm/launchpad-codehosting/templates/launchpad-codehosting-lazr.conf.j2 new file mode 100644 index 0000000..e3e0abe --- /dev/null +++ b/charm/launchpad-codehosting/templates/launchpad-codehosting-lazr.conf.j2 @@ -0,0 +1,29 @@ +# Public configuration data. The contents of this file may be freely shared +# with developers if needed for debugging. + +# A schema's sections, keys, and values are automatically inherited, +# except for '.optional' sections. Update this config to override key +# values. Values are strings, except for numbers that look like ints. +# The tokens true, false, and none are treated as True, False, and None. + +{% from "macros.j2" import opt -%} + +[meta] +extends: ../launchpad-db-lazr.conf + +[launchpad] +launch: False + +[error_reports] +oops_prefix: {{ service_oops_prefix }} + +[codehosting] +access_log: {{ logs_dir }}/{{ service_access_log_file }} +blacklisted_hostnames: {{ blocklisted_hostnames }} +host_key_pair_path: /srv/{{ domain_bzr }}/keys/ +{{- opt("internal_branch_by_id_root", internal_branch_by_id_root) }} +mirrored_branches_root: /srv/{{ domain_bzr }}/mirrors +port: tcp:{{ service_sftp_port }} +rewrite_script_log_file: {{ logs_dir }}/rewrite.log +web_status_port: tcp:{{ service_web_status_port }} + diff --git a/charm/launchpad-codehosting/templates/launchpad-codehosting-secrets-lazr.conf.j2 b/charm/launchpad-codehosting/templates/launchpad-codehosting-secrets-lazr.conf.j2 new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/charm/launchpad-codehosting/templates/launchpad-codehosting-secrets-lazr.conf.j2 diff --git a/charm/launchpad-codehosting/templates/logrotate.conf.j2 b/charm/launchpad-codehosting/templates/logrotate.conf.j2 new file mode 100644 index 0000000..1b87f30 --- /dev/null +++ b/charm/launchpad-codehosting/templates/logrotate.conf.j2 @@ -0,0 +1,16 @@ +{{ logs_dir }}/puller.log {{ logs_dir }}/rewrite.log { + rotate 30 + daily + dateext + compress + delaycompress +} + +{{ logs_dir }}/codehosting-*-access.log {{ logs_dir }}/process-job-source.*.log { + rotate 90 + daily + dateext + compress + delaycompress +} + diff --git a/charm/launchpad-codehosting/templates/rewrite_wrapper.sh.j2 b/charm/launchpad-codehosting/templates/rewrite_wrapper.sh.j2 new file mode 100644 index 0000000..fab9c16 --- /dev/null +++ b/charm/launchpad-codehosting/templates/rewrite_wrapper.sh.j2 @@ -0,0 +1,3 @@ +#!/bin/sh +sudo -u {{ user }} LPCONFIG=launchpad-codehosting {{ code_dir }}/scripts/branch-rewrite.py + diff --git a/charm/launchpad-codehosting/templates/robots.txt b/charm/launchpad-codehosting/templates/robots.txt new file mode 100644 index 0000000..e62cb38 --- /dev/null +++ b/charm/launchpad-codehosting/templates/robots.txt @@ -0,0 +1,4 @@ +# go away +User-agent: * +Disallow: / + diff --git a/charm/launchpad-codehosting/templates/vhosts/bazaar_http.conf.j2 b/charm/launchpad-codehosting/templates/vhosts/bazaar_http.conf.j2 new file mode 100644 index 0000000..ff53bf2 --- /dev/null +++ b/charm/launchpad-codehosting/templates/vhosts/bazaar_http.conf.j2 @@ -0,0 +1,66 @@ +<VirtualHost *:80> + ServerName {{ domain_bzr }} + + CustomLog /var/log/apache2/{{ domain_bzr }}-access.log combined_timer + ErrorLog /var/log/apache2/{{ domain_bzr }}-error.log + + ProxyRequests off + <Proxy *> + <RequireAll> + Require all granted + # Cowboy to stop Sogou web spider smashing bazaar, 13-03-18 + Require expr %{HTTP_USER_AGENT} !~ /^Sogou web spider/ + </RequireAll> + + ErrorDocument 500 /offline.html + ErrorDocument 502 /offline.html + ErrorDocument 503 /offline.html + </Proxy> + ProxyTimeout 20 + + Alias /robots.txt {{ static_files_root }}/robots.txt + Alias /favicon.ico {{ static_files_root }}/images/favicon.ico + Alias /offline.html {{ static_files_root }}/offline.html + Alias /static {{ static_files_root }}/static + <Directory {{ static_files_root }}/www> + Options -Indexes +SymLinksIfOwnerMatch + AllowOverride None + Require all granted + </Directory> + + # Rewrite Logic Flow: + # (1) People hitting the frontpage (i.e. /) are redirected to the + # Launchpad frontpage + # (2) The /favicon.ico file is served normally. + # (3) The /robots.txt file is served normally. + # (4) The /offline.html file is served normally. + # (5) Any /static/* files are served normally. + # (6) Everything else is passed through the branch-rewrite wrapper script + + RewriteEngine On + RewriteMap branch-rewrite prg:{{ scripts_dir }}/rewrite_wrapper.sh + + # (1) / -> Launchpad frontpage + RewriteRule ^/$ https://{{ domain }} [L] + # (2) The /favicon.ico file is served normally. + RewriteRule ^/favicon.ico$ - [L] + # (3) The /robots.txt file is served normally. + RewriteRule ^/robots.txt$ - [L] + # (4) The /offline.html file is served normally. + RewriteRule ^/offline.html$ - [L] + # (5) Any /static/* files are served normally. + RewriteRule ^/static/(.*)$ - [L] + + # (6) The branch-rewrite wrapper script. This sets the appropriate ENV + # setting(s) necessary. + RewriteMap escape int:escape + RewriteMap unescape int:unescape + RewriteRule ^(/.*)$ ${unescape:${branch-rewrite:${escape:$1}}} [L,P] + + # https://portal.admin.canonical.com/C79782 yes this looks mad but it + # is just making NOOP proxypass commands so + # the rewrite [P] lines have persistent pools to work with. + ProxyPass / ! + ProxyPass / http://{{ domain_bzr }}/ +</VirtualHost> + diff --git a/charm/launchpad-codehosting/templates/vhosts/bazaar_https.conf.j2 b/charm/launchpad-codehosting/templates/vhosts/bazaar_https.conf.j2 new file mode 100644 index 0000000..414d7d9 --- /dev/null +++ b/charm/launchpad-codehosting/templates/vhosts/bazaar_https.conf.j2 @@ -0,0 +1,38 @@ +<VirtualHost *:8080> + ServerName {{ domain_bzr }} + ServerAdmin webmas...@launchpad.net + + CustomLog /var/log/apache2/{{ domain_bzr }}-access.log combined_timer + ErrorLog /var/log/apache2/{{ domain_bzr }}-error.log + + # This virtual host serves a few files/statics only; all else is reverse + # proxy to codebrowse. + DocumentRoot {{ static_files_root }} + + <Directory {{ static_files_root }}> + Options -Indexes +SymLinksIfOwnerMatch + Require all granted + </Directory> + + ProxyRequests off + <Proxy *> + <RequireAll> + Require all granted + # Cowboy to stop Sogou web spider smashing bazaar, 13-03-18 + Require expr %{HTTP_USER_AGENT} !~ /^Sogou web spider/ + </RequireAll> + ErrorDocument 500 /offline.html + ErrorDocument 502 /offline.html + ErrorDocument 503 /offline.html + </Proxy> + ProxyTimeout 20 + ProxyPassReverse / {{ codebrowse_internal_endpoint }} + + RewriteEngine On + RewriteRule ^/offline.html$ - [L] + RewriteRule ^/robots.txt$ - [L] + RewriteRule ^/favicon.ico$ /static/images/favicon.ico [L] + RewriteRule ^/static/(.*)$ - [L] + RewriteRule ^/(.*)$ {{ codebrowse_internal_endpoint }}/$1 [P,L] +</VirtualHost> + diff --git a/charm/launchpad-codehosting/templates/vhosts/bazaar_internal_branch_by_id.conf.j2 b/charm/launchpad-codehosting/templates/vhosts/bazaar_internal_branch_by_id.conf.j2 new file mode 100644 index 0000000..a92f2fd --- /dev/null +++ b/charm/launchpad-codehosting/templates/vhosts/bazaar_internal_branch_by_id.conf.j2 @@ -0,0 +1,18 @@ +<VirtualHost *:8081> + ServerName {{ domain_bzr_internal }} + + CustomLog /var/log/apache2/{{ domain_bzr_internal }}-access.log combined_timer + ErrorLog /var/log/apache2/{{ domain_bzr_internal }}-error.log + + # Default location is for all the .bzr repositories in mirrors/* + # We serve .bzr/* via http or bzr+ssh; not https + DocumentRoot {{ bzr_repositories_root }} + + <Directory {{ bzr_repositories_root }}> + Options SymLinksIfOwnerMatch Indexes + AllowOverride None + Require all granted + </Directory> + +</VirtualHost> + diff --git a/charm/launchpad-codehosting/templates/vhosts/common.conf b/charm/launchpad-codehosting/templates/vhosts/common.conf new file mode 100644 index 0000000..bea1e98 --- /dev/null +++ b/charm/launchpad-codehosting/templates/vhosts/common.conf @@ -0,0 +1,6 @@ +# Needed outside VirtualHost config +LogFormat "%h %D %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined_timer + +# For debugging ONLY - do not turn on in normal operation +# LogLevel alert rewrite:trace4 +
_______________________________________________ Mailing list: https://launchpad.net/~launchpad-reviewers Post to : launchpad-reviewers@lists.launchpad.net Unsubscribe : https://launchpad.net/~launchpad-reviewers More help : https://help.launchpad.net/ListHelp