masatana opened a new pull request, #1313:
URL: https://github.com/apache/bigtop/pull/1313
<!--
Thanks for sending a pull request!
1. If this is your first time, please read our contributor guidelines:
https://cwiki.apache.org/confluence/display/BIGTOP/How+to+Contribute
2. Make sure your PR title starts with JIRA issue id, e.g.,
'BIGTOP-3638: Your PR title ...'.
-->
### Description of PR
https://issues.apache.org/jira/browse/BIGTOP-4195
Introduced systemd unit files for Hadoop (MapReduce History Server)
Add some environment variable to allow `/usr/bin/mapred` to control the
hadoop-mapreduce-historyserver.
Use `RuntimeDirectory` setting to prepare the `/run/hadoop-mapreduce`
directory instead of using systemd-tmpfile
### How was this patch tested?
Tested on both RPM(rocky linux8) and DEB (Ubuntu 24.04 w/ #1304, #1305)
* Check if building package is SUCCESSFUL
* Check if hadoop-mapreduce-historyserver is running with the packages
* Check if `/run/hadoop-mapreduce` directory exists after restarting
<details>
<summary>Tested logs</summary>
#### RPM
```
$ ./gradlew allclean hadoop-pkg-ind repo-ind -POS=rockylinux-8
$ cd provisioner/docker
$ ./docker-hadoop.sh --enable-local-repo --disable-gpg-check
--docker-compose-plugin -C config_rockylinux-8.yaml -F
docker-compose-cgroupv2.yml --stack hdfs,yarn,mapreduce -c 1
$ ./docker-hadoop.sh -dcp --exec 1 /bin/bash
# check if the systemd unit file is used
[root@1413986eab28 /]# systemctl status hadoop-mapreduce-historyserver
● hadoop-mapreduce-historyserver.service - Hadoop MapReduce HistoryServer
Loaded: loaded
(/usr/lib/systemd/system/hadoop-mapreduce-historyserver.service; static; vendor
preset: disabled)
Active: active (running) since Thu 2024-11-21 00:02:29 UTC; 1min 57s ago
Docs: https://hadoop.apache.org/
Main PID: 7024 (java)
Tasks: 59 (limit: 5099)
Memory: 326.3M
CGroup: /system.slice/hadoop-mapreduce-historyserver.service
└─7024
/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.432.b06-2.el8.aarch64/bin/java
-Dproc_historyserver -Djava.net.preferIPv4Stack=true
-Dmapred.jobsummary.logger=INFO,RFA -Dyarn.log.dir=/var/log/hadoop-mapreduce
-Dyarn.log.file=hadoop-mapred-historyserver-1413986eab28.log
-Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console
-Djava.library.p>
Nov 21 00:02:27 1413986eab28 systemd[1]: Starting Hadoop MapReduce
HistoryServer...
Nov 21 00:02:29 1413986eab28 systemd[1]: Started Hadoop MapReduce
HistoryServer.
[root@1413986eab28 /]# systemctl cat hadoop-mapreduce-historyserver
# /usr/lib/systemd/system/hadoop-mapreduce-historyserver.service
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[Unit]
Documentation=https://hadoop.apache.org/
Description=Hadoop MapReduce HistoryServer
Before=multi-user.target
Before=graphical.target
After=remote-fs.target
[Service]
User=mapred
Group=mapred
Type=forking
Restart=no
TimeoutSec=5min
IgnoreSIGPIPE=no
KillMode=process
GuessMainPID=no
RemainAfterExit=no
PIDFile=/run/hadoop-mapreduce/hadoop-mapred-historyserver.pid
RuntimeDirectory=hadoop-mapreduce
SuccessExitStatus=5 6
ExecStart=/usr/bin/mapred --config /etc/hadoop/conf --daemon start
historyserver
ExecStop=/usr/bin/mapred --config /etc/hadoop/conf --daemon stop
historyserver
[root@1413986eab28 /]# systemctl restart hadoop-mapreduce-historyserver
[root@1413986eab28 /]# systemctl status hadoop-mapreduce-historyserver
● hadoop-mapreduce-historyserver.service - Hadoop MapReduce HistoryServer
Loaded: loaded
(/usr/lib/systemd/system/hadoop-mapreduce-historyserver.service; static; vendor
preset: disabled)
Active: active (running) since Thu 2024-11-21 00:04:56 UTC; 2s ago
Docs: https://hadoop.apache.org/
Process: 8487 ExecStop=/usr/bin/mapred --config /etc/hadoop/conf --daemon
stop historyserver (code=exited, status=0/SUCCESS)
Process: 8544 ExecStart=/usr/bin/mapred --config /etc/hadoop/conf --daemon
start historyserver (code=exited, status=0/SUCCESS)
Main PID: 8596 (java)
Tasks: 62 (limit: 5099)
Memory: 402.8M
CGroup: /system.slice/hadoop-mapreduce-historyserver.service
└─8596
/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.432.b06-2.el8.aarch64/bin/java
-Dproc_historyserver -Djava.net.preferIPv4Stack=true
-Dmapred.jobsummary.logger=INFO,RFA -Dyarn.log.dir=/var/log/hadoop-mapreduce
-Dyarn.log.file=hadoop-mapred-historyserver-1413986eab28.log
-Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console
-Djava.library.p>
Nov 21 00:04:54 1413986eab28 systemd[1]: Starting Hadoop MapReduce
HistoryServer...
Nov 21 00:04:56 1413986eab28 systemd[1]: Started Hadoop MapReduce
HistoryServer.
[root@1413986eab28 /]# exit
# restarting the container
$ docker ps
CONTAINER ID IMAGE COMMAND
CREATED STATUS PORTS NAMES
1413986eab28 bigtop/puppet:trunk-rockylinux-8-aarch64 "/sbin/init" 8
minutes ago Up 8 minutes 20241121_085649_r5067-bigtop-1
$ docker restart 1413986eab28
1413986eab28
$ ./docker-hadoop.sh -dcp --exec 1 systemctl status
hadoop-mapreduce-historyserver
● hadoop-mapreduce-historyserver.service - Hadoop MapReduce HistoryServer
Loaded: loaded
(/usr/lib/systemd/system/hadoop-mapreduce-historyserver.service; static; vendor
preset: disabled)
Active: inactive (dead)
Docs: https://hadoop.apache.org/
$ ./docker-hadoop.sh -dcp --exec 1 systemctl start
hadoop-mapreduce-historyserver
$ ./docker-hadoop.sh -dcp --exec 1 systemctl status
hadoop-mapreduce-historyserver
● hadoop-mapreduce-historyserver.service - Hadoop MapReduce HistoryServer
Loaded: loaded
(/usr/lib/systemd/system/hadoop-mapreduce-historyserver.service; static; vendor
preset: disabled)
Active: active (running) since Thu 2024-11-21 00:06:05 UTC; 2s ago
Docs: https://hadoop.apache.org/
Process: 1106 ExecStart=/usr/bin/mapred --config /etc/hadoop/conf --daemon
start historyserver (code=exited, status=0/SUCCESS)
Main PID: 1159 (java)
Tasks: 62 (limit: 5099)
Memory: 354.3M
CGroup: /system.slice/hadoop-mapreduce-historyserver.service
└─1159
/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.432.b06-2.el8.aarch64/bin/java
-Dproc_historyserver -Djava.net.preferIPv4Stack=true
-Dmapred.jobsummary.logger=INFO,RFA -Dyarn.log.dir=/var/log/hadoop-mapreduce
-Dyarn.log.file=hadoop-mapred-historyserver-1413986eab28.log
-Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console
-Djava.library.p>
Nov 21 00:06:03 1413986eab28 systemd[1]: Starting Hadoop MapReduce
HistoryServer...
Nov 21 00:06:05 1413986eab28 systemd[1]: Started Hadoop MapReduce
HistoryServer.
```
#### DEB
After applying #1304, #1305 and built the ubuntu 24.04 container
```
$ ./gradlew hadoop-clean bigtop-utils-pkg bigtop-jsvc-pkg bigtop-groovy-pkg
hadoop-pkg repo
$ cd provisioner/docker
$ ./docker-hadoop.sh --enable-local-repo --disable-gpg-check
--docker-compose-plugin -C config_ubuntu-24.04.yaml -F
docker-compose-cgroupv2.yml --stack hdfs,yarn,mapreduce -c 1
# check if the systemd unit file is used
ubuntu@ip-172-31-2-106:~/bigtop/provisioner/docker$ ./docker-hadoop.sh -dcp
--exec 1 systemctl cat hadoop-mapreduce-historyserver
# /usr/lib/systemd/system/hadoop-mapreduce-historyserver.service
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[Unit]
Documentation=https://hadoop.apache.org/
Description=Hadoop MapReduce HistoryServer
Before=multi-user.target
Before=graphical.target
After=remote-fs.target
[Service]
User=mapred
Group=mapred
Type=forking
Restart=no
TimeoutSec=5min
IgnoreSIGPIPE=no
KillMode=process
GuessMainPID=no
RemainAfterExit=no
PIDFile=/run/hadoop-mapreduce/hadoop-mapred-historyserver.pid
RuntimeDirectory=hadoop-mapreduce
SuccessExitStatus=5 6
ExecStart=/usr/bin/mapred --config /etc/hadoop/conf --daemon start
historyserver
ExecStop=/usr/bin/mapred --config /etc/hadoop/conf --daemon stop
historyserver
# restarting the container
ubuntu@ip-172-31-2-106:~/bigtop/provisioner/docker$ docker ps
CONTAINER ID IMAGE COMMAND CREATED
STATUS PORTS NAMES
efc631471eb7 bigtop/puppet:trunk-ubuntu-24.04 "/sbin/init" 5 minutes
ago Up 5 minutes 20241123_025918_r30555-bigtop-1
ubuntu@ip-172-31-2-106:~/bigtop/provisioner/docker$ docker restart
efc631471eb7
efc631471eb7
ubuntu@ip-172-31-2-106:~/bigtop/provisioner/docker$ ./docker-hadoop.sh -dcp
--exec 1 /bin/bash
root@efc631471eb7:/# systemctl start hadoop-mapreduce-historyserver
root@efc631471eb7:/# systemctl status hadoop-mapreduce-historyserver
● hadoop-mapreduce-historyserver.service - Hadoop MapReduce HistoryServer
Loaded: loaded
(/usr/lib/systemd/system/hadoop-mapreduce-historyserver.service; static)
Active: active (running) since Sat 2024-11-23 03:05:43 UTC; 11s ago
Docs: https://hadoop.apache.org/
Process: 1024 ExecStart=/usr/bin/mapred --config /etc/hadoop/conf
--daemon start historyserver (code=exited, status=0/SUCCESS)
Main PID: 1064 (java)
Tasks: 54 (limit: 1389)
Memory: 288.9M (peak: 300.8M)
CPU: 6.609s
CGroup: /system.slice/hadoop-mapreduce-historyserver.service
└─1064 /usr/lib/jvm/java-1.8.0-openjdk-amd64/bin/java
-Dproc_historyserver -Djava.net.preferIPv4Stack=true
-Dmapred.jobsummary.logger=INFO,RFA -Dyarn.log.dir=/var/log/hadoop-mapreduce
-Dyarn.log.file=hadoop-mapred-historyserver-efc631471eb7.log
-Dyarn.home.dir=/usr/lib/hadoop-yarn -Dyarn.root.logger=INFO,console
-Djava.library.path=//usr/lib/hadoop/lib/native
-Dhadoop.log.dir=/var/log/hadoop-mapreduce
-Dhadoop.log.file=hadoop-mapred-historyserver-efc631471eb7.log
-Dhadoop.home.dir=//usr/lib/hadoop -Dhadoop.id.str=mapred
-Dhadoop.root.logger=INFO,RFA -Dhadoop.policy.file=hadoop-policy.xml
-Dhadoop.security.logger=INFO,NullAppender
org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer
Nov 23 03:05:41 efc631471eb7 systemd[1]: Starting
hadoop-mapreduce-historyserver.service - Hadoop MapReduce HistoryServer...
Nov 23 03:05:43 efc631471eb7 systemd[1]: Started
hadoop-mapreduce-historyserver.service - Hadoop MapReduce HistoryServer.
root@efc631471eb7:/# ls /run/hadoop-mapreduce/
hadoop-mapred-historyserver.pid
root@efc631471eb7:/#
```
</details>
### For code changes:
- [x] Does the title or this PR starts with the corresponding JIRA issue id
(e.g. 'BIGTOP-3638. Your PR title ...')?
- [x] Make sure that newly added files do not have any licensing issues.
When in doubt refer to https://www.apache.org/licenses/
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
