Hi,
I have setup a testbed to try orangefs 2.9.3 using Infiniband interface.
When I tried to mount this orangefs
mount -t pvfs2 ib://hbcogfs001:3335/orangefs /mnt/orangefs
The strace has shown that the mount stopped at
>>>> mount("ib://hbcogfs001:3335/orangefs", "/mnt/orangefs", "pvfs2",
>>>> MS_MGC_VAL, NULL
Cloud anyone give me some suggestion about mounting through IB interface ?
Thanks.
HB Chen
HPC-5
Los Alamos National Lab
[email protected]
505-665-3591
[1]
I used Mellanox Infiniband QDR NIC and Qdr IB switch
ib0 port is ACTIVE
[root@hbcogfs001 orangefs-2.9.3]# ibstat
CA 'mlx4_0'
CA type: MT4099
Number of ports: 2
Firmware version: 2.35.5100
Hardware version: 1
Node GUID: 0xf4521403008a36f0
System image GUID: 0xf4521403008a36f3
Port 1:
State: Active
Physical state: LinkUp
Rate: 40
Base lid: 6
LMC: 0
SM lid: 1
Capability mask: 0x02514868
Port GUID: 0xf4521403008a36f1
Link layer: InfiniBand
Port 2:
State: Down
Physical state: Polling
Rate: 10
Base lid: 0
LMC: 0
SM lid: 0
Capability mask: 0x02514868
Port GUID: 0xf4521403008a36f2
Link layer: InfiniBand
[root@hbcogfs001 orangefs-2.9.3]#
[root@hbcogfs001 orangefs-2.9.3]# ifconfig ib0
Ifconfig uses the ioctl access method to get the full address information,
which limits hardware addresses to 8 bytes.
Because Infiniband address has 20 bytes, only the first 8 bytes are displayed
correctly.
Ifconfig is obsolete! For replacement check ip.
ib0 Link encap:InfiniBand HWaddr
A0:00:02:20:FE:80:00:00:00:00:00:00:00:00:00:00:00:00:00:00
inet addr:10.0.1.1 Bcast:10.0.1.255 Mask:255.255.255.0
inet6 addr: fe80::f652:1403:8a:36f1/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:65520 Metric:1
RX packets:134 errors:0 dropped:0 overruns:0 frame:0
TX packets:157 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:10000
RX bytes:12745 (12.4 KiB) TX bytes:14525 (14.1 KiB)
[2]
orange server and client processes
root@hbcogfs001 orangefs-2.9.3]# ps -e | grep pvfs
24808 ? 00:00:09 pvfs2-server
24877 ? 00:00:00 pvfs2-client
24878 ? 00:00:00 pvfs2-client-co
[3] server config file
[root@hbcogfs001 orangefs-2.9.3]# more /opt/orangefs/etc/orangefs-server.conf
<Defaults>
UnexpectedRequests 50
EventLogging none
EnableTracing no
LogStamp datetime
BMIModules bmi_ib
FlowModules flowproto_multiqueue
PerfUpdateInterval 1000
ServerJobBMITimeoutSecs 30
ServerJobFlowTimeoutSecs 30
ClientJobBMITimeoutSecs 300
ClientJobFlowTimeoutSecs 300
ClientRetryLimit 5
ClientRetryDelayMilliSecs 2000
PrecreateBatchSize 0,32,512,32,32,32,0
PrecreateLowThreshold 0,16,256,16,16,16,0
DataStorageSpace /mnt/tmpfs/data
MetadataStorageSpace /mnt/tmpfs/meta
LogFile /var/log/orangefs-server.log
</Defaults>
<Aliases>
Alias hbcogfs001 ib://hbcogfs001:3335
</Aliases>
<Filesystem>
Name orangefs
ID 1413148739
RootHandle 1048576
FileStuffing yes
DistrDirServersInitial 1
DistrDirServersMax 1
DistrDirSplitSize 100
<MetaHandleRanges>
Range hbcogfs001 3-4611686018427387904
</MetaHandleRanges>
<DataHandleRanges>
Range hbcogfs001 4611686018427387905-9223372036854775806
</DataHandleRanges>
<StorageHints>
TroveSyncMeta yes
TroveSyncData no
TroveMethod alt-aio
</StorageHints>
</Filesystem>
[root@hbcogfs001 orangefs-2.
/etc/pvfs2tab
[root@hbcogfs001 orangefs-2.9.3]# more /etc/pvfs2tab
ib://hbcogfs001:3335/orangefs /mnt/orangefs pvfs2 defaults,noauto 0 0
orangefs : configure
./configure
--prefix=/opt/orangefs
--with-kernel=/usr/src/kernels/2.6.32-573.12.1.el6.x86_64
--with-openib=/usr
--without-bmi-tcp
--with-db=/usr/local/src/BerkeleyDB/
[5]
// strace log
[root@hbcogfs001 orangesfs]# strace mount -t pvfs2
ib://hbcogfs001:3335/orangefs /mnt/orangefs
execve("/bin/mount", ["mount", "-t", "pvfs2", "ib://hbcogfs001:3335/orangefs",
"/mnt/orangefs"], [/* 42 vars */]) = 0
brk(0) = 0x7fca5ab36000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x7fca5a1d2000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/usr/local/src/BerkeleyDB/lib/tls/x86_64/libblkid.so.1", O_RDONLY) = -1
ENOENT (No such file or directory)
stat("/usr/local/src/BerkeleyDB/lib/tls/x86_64", 0x7ffc44087160) = -1 ENOENT
(No such file or directory)
open("/usr/local/src/BerkeleyDB/lib/tls/libblkid.so.1", O_RDONLY) = -1 ENOENT
(No such file or directory)
stat("/usr/local/src/BerkeleyDB/lib/tls", 0x7ffc44087160) = -1 ENOENT (No such
file or directory)
open("/usr/local/src/BerkeleyDB/lib/x86_64/libblkid.so.1", O_RDONLY) = -1
ENOENT (No such file or directory)
stat("/usr/local/src/BerkeleyDB/lib/x86_64", 0x7ffc44087160) = -1 ENOENT (No
such file or directory)
open("/usr/local/src/BerkeleyDB/lib/libblkid.so.1", O_RDONLY) = -1 ENOENT (No
such file or directory)
stat("/usr/local/src/BerkeleyDB/lib", {st_mode=S_IFDIR|0755, st_size=4096,
...}) = 0
open("/usr/local/src/openmpi/lib/tls/x86_64/libblkid.so.1", O_RDONLY) = -1
ENOENT (No such file or directory)
stat("/usr/local/src/openmpi/lib/tls/x86_64", 0x7ffc44087160) = -1 ENOENT (No
such file or directory)
open("/usr/local/src/openmpi/lib/tls/libblkid.so.1", O_RDONLY) = -1 ENOENT (No
such file or directory)
stat("/usr/local/src/openmpi/lib/tls", 0x7ffc44087160) = -1 ENOENT (No such
file or directory)
open("/usr/local/src/openmpi/lib/x86_64/libblkid.so.1", O_RDONLY) = -1 ENOENT
(No such file or directory)
stat("/usr/local/src/openmpi/lib/x86_64", 0x7ffc44087160) = -1 ENOENT (No such
file or directory)
open("/usr/local/src/openmpi/lib/libblkid.so.1", O_RDONLY) = -1 ENOENT (No such
file or directory)
stat("/usr/local/src/openmpi/lib", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
open("/opt/orangefs/lib/tls/x86_64/libblkid.so.1", O_RDONLY) = -1 ENOENT (No
such file or directory)
stat("/opt/orangefs/lib/tls/x86_64", 0x7ffc44087160) = -1 ENOENT (No such file
or directory)
open("/opt/orangefs/lib/tls/libblkid.so.1", O_RDONLY) = -1 ENOENT (No such file
or directory)
stat("/opt/orangefs/lib/tls", 0x7ffc44087160) = -1 ENOENT (No such file or
directory)
open("/opt/orangefs/lib/x86_64/libblkid.so.1", O_RDONLY) = -1 ENOENT (No such
file or directory)
stat("/opt/orangefs/lib/x86_64", 0x7ffc44087160) = -1 ENOENT (No such file or
directory)
open("/opt/orangefs/lib/libblkid.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
stat("/opt/orangefs/lib", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
open("/usr/lib64/tls/x86_64/libblkid.so.1", O_RDONLY) = -1 ENOENT (No such file
or directory)
stat("/usr/lib64/tls/x86_64", 0x7ffc44087160) = -1 ENOENT (No such file or
directory)
open("/usr/lib64/tls/libblkid.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
stat("/usr/lib64/tls", {st_mode=S_IFDIR|0555, st_size=4096, ...}) = 0
open("/usr/lib64/x86_64/libblkid.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
stat("/usr/lib64/x86_64", 0x7ffc44087160) = -1 ENOENT (No such file or
directory)
open("/usr/lib64/libblkid.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
stat("/usr/lib64", {st_mode=S_IFDIR|0555, st_size=135168, ...}) = 0
open("tls/x86_64/libblkid.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("tls/libblkid.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("x86_64/libblkid.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("libblkid.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=139824, ...}) = 0
mmap(NULL, 139824, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fca5a1af000
close(3) = 0
open("/lib64/libblkid.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260Z`\"7\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=138616, ...}) = 0
mmap(NULL, 2231272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7fca59d93000
mprotect(0x7fca59db1000, 2097152, PROT_NONE) = 0
mmap(0x7fca59fb1000, 12288, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e000) = 0x7fca59fb1000
close(3) = 0
open("/usr/local/src/BerkeleyDB/lib/libuuid.so.1", O_RDONLY) = -1 ENOENT (No
such file or directory)
open("/usr/local/src/openmpi/lib/libuuid.so.1", O_RDONLY) = -1 ENOENT (No such
file or directory)
open("/opt/orangefs/lib/libuuid.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("/usr/lib64/tls/libuuid.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("/usr/lib64/libuuid.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("tls/x86_64/libuuid.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("tls/libuuid.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("x86_64/libuuid.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("libuuid.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libuuid.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\25 #7\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=18936, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x7fca5a1ae000
mmap(NULL, 2111272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7fca59b8f000
mprotect(0x7fca59b93000, 2093056, PROT_NONE) = 0
mmap(0x7fca59d92000, 4096, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7fca59d92000
close(3) = 0
open("/usr/local/src/BerkeleyDB/lib/libselinux.so.1", O_RDONLY) = -1 ENOENT (No
such file or directory)
open("/usr/local/src/openmpi/lib/libselinux.so.1", O_RDONLY) = -1 ENOENT (No
such file or directory)
open("/opt/orangefs/lib/libselinux.so.1", O_RDONLY) = -1 ENOENT (No such file
or directory)
open("/usr/lib64/tls/libselinux.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("/usr/lib64/libselinux.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("tls/x86_64/libselinux.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("tls/libselinux.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("x86_64/libselinux.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("libselinux.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libselinux.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320X`\0357\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=124624, ...}) = 0
mmap(NULL, 2221912, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7fca59970000
mprotect(0x7fca5998d000, 2093056, PROT_NONE) = 0
mmap(0x7fca59b8c000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c000) = 0x7fca59b8c000
mmap(0x7fca59b8e000, 1880, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fca59b8e000
close(3) = 0
open("/usr/local/src/BerkeleyDB/lib/libsepol.so.1", O_RDONLY) = -1 ENOENT (No
such file or directory)
open("/usr/local/src/openmpi/lib/libsepol.so.1", O_RDONLY) = -1 ENOENT (No such
file or directory)
open("/opt/orangefs/lib/libsepol.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("/usr/lib64/tls/libsepol.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("/usr/lib64/libsepol.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("tls/x86_64/libsepol.so.1", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("tls/libsepol.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("x86_64/libsepol.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("libsepol.so.1", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libsepol.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200>\34017\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=251112, ...}) = 0
mmap(NULL, 2345600, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7fca59733000
mprotect(0x7fca5976e000, 2097152, PROT_NONE) = 0
mmap(0x7fca5996e000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3b000) = 0x7fca5996e000
close(3) = 0
open("/usr/local/src/BerkeleyDB/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such
file or directory)
open("/usr/local/src/openmpi/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such
file or directory)
open("/opt/orangefs/lib/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("/usr/lib64/tls/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("/usr/lib64/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("tls/x86_64/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("tls/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("x86_64/libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("libc.so.6", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libc.so.6", O_RDONLY) = 3
read(3,
"\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\356\341\0337\0\0\0"..., 832) =
832
fstat(3, {st_mode=S_IFREG|0755, st_size=1926520, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x7fca5a1ad000
mmap(NULL, 3750152, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7fca5939f000
mprotect(0x7fca59529000, 2097152, PROT_NONE) = 0
mmap(0x7fca59729000, 20480, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18a000) = 0x7fca59729000
mmap(0x7fca5972e000, 18696, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fca5972e000
close(3) = 0
open("/usr/local/src/BerkeleyDB/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such
file or directory)
open("/usr/local/src/openmpi/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such
file or directory)
open("/opt/orangefs/lib/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("/usr/lib64/tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or
directory)
open("/usr/lib64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("tls/x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("tls/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("x86_64/libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("libdl.so.2", O_RDONLY) = -1 ENOENT (No such file or directory)
open("/lib64/libdl.so.2", O_RDONLY) = 3
read(3,
"\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340\r\240\0347\0\0\0"..., 832)
= 832
fstat(3, {st_mode=S_IFREG|0755, st_size=22536, ...}) = 0
mmap(NULL, 2109696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) =
0x7fca5919b000
mprotect(0x7fca5919d000, 2097152, PROT_NONE) = 0
mmap(0x7fca5939d000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fca5939d000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x7fca5a1ac000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x7fca5a1aa000
arch_prctl(ARCH_SET_FS, 0x7fca5a1aa7e0) = 0
mprotect(0x7fca5939d000, 4096, PROT_READ) = 0
mprotect(0x7fca59729000, 16384, PROT_READ) = 0
mprotect(0x7fca5996e000, 4096, PROT_READ) = 0
mprotect(0x7fca59b8c000, 4096, PROT_READ) = 0
mprotect(0x7fca5a1d3000, 4096, PROT_READ) = 0
munmap(0x7fca5a1af000, 139824) = 0
statfs("/selinux", {f_type="EXT2_SUPER_MAGIC", f_bsize=4096, f_blocks=38717175,
f_bfree=36062226, f_bavail=34093830, f_files=9846784, f_ffree=9521173,
f_fsid={-1678995328, -1524924415}, f_namelen=255, f_frsize=4096}) = 0
brk(0) = 0x7fca5ab36000
brk(0x7fca5ab57000) = 0x7fca5ab57000
open("/proc/filesystems", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) =
0x7fca5a1d1000
read(3, "nodev\tsysfs\nnodev\trootfs\nnodev\tb"..., 1024) = 377
read(3, "", 1024) = 0
close(3) = 0
munmap(0x7fca5a1d1000, 4096) = 0
open("/usr/lib/locale/locale-archive", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=99158576, ...}) = 0
mmap(NULL, 99158576, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fca5330a000
close(3) = 0
umask(022) = 022
open("/dev/null", O_RDWR) = 3
close(3) = 0
getuid() = 0
geteuid() = 0
getcwd("/usr/local/src/orangesfs", 4095) = 25
readlink("/usr/local/src/orangesfs/ib:", 0x7ffc44085720, 4096) = -1 ENOENT (No
such file or directory)
stat("/sbin/mount.pvfs2", 0x7ffc44087310) = -1 ENOENT (No such file or
directory)
rt_sigprocmask(SIG_BLOCK, ~[TRAP SEGV RTMIN RT_1], NULL, 8) = 0
stat("/sbin/mount.pvfs2", 0x7ffc440872e0) = -1 ENOENT (No such file or
directory)
mount("ib://hbcogfs001:3335/orangefs", "/mnt/orangefs", "pvfs2", MS_MGC_VAL,
NULL
_______________________________________________
Pvfs2-users mailing list
[email protected]
http://www.beowulf-underground.org/mailman/listinfo/pvfs2-users
