[ClusterLabs] Upgrade corosync problem
Salvatore D'angelo
sasadangelo at gmail.com
Fri Jul 6 08:29:50 EDT 2018
Here some strike of corosync:
execve("/usr/sbin/corosync", ["corosync"], [/* 21 vars */]) = 0
brk(0) = 0x563b1f774000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=26561, ...}) = 0
mmap(NULL, 26561, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f0cd4182000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/libtotem_pg.so.5", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260a\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=917346, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0cd4181000
mmap(NULL, 2267392, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0cd3d3d000
mprotect(0x7f0cd3d61000, 2093056, PROT_NONE) = 0
mmap(0x7f0cd3f60000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x23000) = 0x7f0cd3f60000
mmap(0x7f0cd3f62000, 18688, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0cd3f62000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/libcorosync_common.so.4", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\6\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=43858, ...}) = 0
mmap(NULL, 2105360, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0cd3b3a000
mprotect(0x7f0cd3b3b000, 2097152, PROT_NONE) = 0
mmap(0x7f0cd3d3b000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x7f0cd3d3b000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\16\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14664, ...}) = 0
mmap(NULL, 2109744, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0cd3936000
mprotect(0x7f0cd3939000, 2093056, PROT_NONE) = 0
mmap(0x7f0cd3b38000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f0cd3b38000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0po\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=141574, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0cd4180000
mmap(NULL, 2217264, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0cd3718000
mprotect(0x7f0cd3731000, 2093056, PROT_NONE) = 0
mmap(0x7f0cd3930000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18000) = 0x7f0cd3930000
mmap(0x7f0cd3932000, 13616, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0cd3932000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P \2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1857312, ...}) = 0
mmap(NULL, 3965632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0cd334f000
mprotect(0x7f0cd350d000, 2097152, PROT_NONE) = 0
mmap(0x7f0cd370d000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1be000) = 0x7f0cd370d000
mmap(0x7f0cd3713000, 17088, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0cd3713000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/libqb.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\243\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=951833, ...}) = 0
mmap(NULL, 6717576, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0cd2ce6000
mprotect(0x7f0cd2d0b000, 6557696, PROT_NONE) = 0
mmap(0x7f0cd2f0a000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x24000) = 0x7f0cd2f0a000
mmap(0x7f0cd2f0c000, 264352, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0cd2f0c000
mmap(0x7f0cd334c000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x66000) = 0x7f0cd334c000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/libnss3.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320h\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=1297656, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0cd417f000
mmap(NULL, 3398792, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0cd29a8000
mprotect(0x7f0cd2ade000, 2097152, PROT_NONE) = 0
mmap(0x7f0cd2cde000, 28672, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x136000) = 0x7f0cd2cde000
mmap(0x7f0cd2ce5000, 3208, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0cd2ce5000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/libnspr4.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\266\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=239440, ...}) = 0
mmap(NULL, 2345184, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0cd276b000
mprotect(0x7f0cd27a4000, 2093056, PROT_NONE) = 0
mmap(0x7f0cd29a3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x38000) = 0x7f0cd29a3000
mmap(0x7f0cd29a5000, 10464, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f0cd29a5000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/libnssutil3.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\267\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=182992, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0cd417e000
mmap(NULL, 2279616, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0cd253e000
mprotect(0x7f0cd2564000, 2093056, PROT_NONE) = 0
mmap(0x7f0cd2763000, 32768, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x25000) = 0x7f0cd2763000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/libplc4.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\23\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=18640, ...}) = 0
mmap(NULL, 2113904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0cd2339000
mprotect(0x7f0cd233d000, 2093056, PROT_NONE) = 0
mmap(0x7f0cd253c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f0cd253c000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/usr/lib/x86_64-linux-gnu/libplds4.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260\r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14448, ...}) = 0
mmap(NULL, 2109696, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0cd2135000
mprotect(0x7f0cd2138000, 2093056, PROT_NONE) = 0
mmap(0x7f0cd2337000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f0cd2337000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/librt.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P#\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=31792, ...}) = 0
mmap(NULL, 2129016, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f0cd1f2d000
mprotect(0x7f0cd1f34000, 2093056, PROT_NONE) = 0
mmap(0x7f0cd2133000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f0cd2133000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0cd417d000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0cd417c000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0cd417a000
arch_prctl(ARCH_SET_FS, 0x7f0cd417a740) = 0
mprotect(0x7f0cd370d000, 16384, PROT_READ) = 0
mprotect(0x7f0cd3930000, 4096, PROT_READ) = 0
mprotect(0x7f0cd2133000, 4096, PROT_READ) = 0
mprotect(0x7f0cd3b38000, 4096, PROT_READ) = 0
mprotect(0x7f0cd29a3000, 4096, PROT_READ) = 0
mprotect(0x7f0cd2337000, 4096, PROT_READ) = 0
mprotect(0x7f0cd253c000, 4096, PROT_READ) = 0
mprotect(0x7f0cd2763000, 28672, PROT_READ) = 0
mprotect(0x7f0cd2cde000, 20480, PROT_READ) = 0
mprotect(0x7f0cd2f0a000, 4096, PROT_READ) = 0
mprotect(0x7f0cd3d3b000, 4096, PROT_READ) = 0
mprotect(0x7f0cd3f60000, 4096, PROT_READ) = 0
mprotect(0x563b1e764000, 4096, PROT_READ) = 0
mprotect(0x7f0cd4189000, 4096, PROT_READ) = 0
munmap(0x7f0cd4182000, 26561) = 0
set_tid_address(0x7f0cd417aa10) = 8532
set_robust_list(0x7f0cd417aa20, 24) = 0
futex(0x7fff877daea0, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 1, NULL, 7f0cd417a740) = -1 EAGAIN (Resource temporarily unavailable)
rt_sigaction(SIGRTMIN, {0x7f0cd371e9f0, [], SA_RESTORER|SA_SIGINFO, 0x7f0cd3728330}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7f0cd371ea80, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f0cd3728330}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
brk(0) = 0x563b1f774000
brk(0x563b1f795000) = 0x563b1f795000
futex(0x7f0cd3b390d0, FUTEX_WAKE_PRIVATE, 2147483647) = 0
write(2, "corosync: main.c:143: logsys_qb_"..., 207corosync: main.c:143: logsys_qb_init: Assertion `"implicit callsite section is populated, otherwise target's build is at fault, preventing reliable logging" && __start___verbose != __stop___verbose' failed.
) = 207
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f0cd4188000
rt_sigprocmask(SIG_UNBLOCK, [ABRT], NULL, 8) = 0
tgkill(8532, 8532, SIGABRT) = 0
--- SIGABRT {si_signo=SIGABRT, si_code=SI_TKILL, si_pid=8532, si_uid=0} ---
+++ killed by SIGABRT +++
Aborted
> On 6 Jul 2018, at 14:24, Salvatore D'angelo <sasadangelo at gmail.com> wrote:
>
> Hi All,
>
> The option --ulimit memlock=536870912 worked fine.
>
> I have now another strange issue. The upgrade without updating libqb (leaving the 0.16.0) worked fine.
> If after the upgrade I stop pacemaker and corosync, I download the latest libqb version:
> https://github.com/ClusterLabs/libqb/releases/download/v1.0.3/libqb-1.0.3.tar.gz <https://github.com/ClusterLabs/libqb/releases/download/v1.0.3/libqb-1.0.3.tar.gz>
> build and install it everything works fine.
>
> If I try to install in sequence (after the installation of old code):
>
> libqb 1.0.3
> corosync 2.4.4
> pacemaker 1.1.18
> crmsh 3.0.1
> resource agents 4.1.1
>
> when I try to start corosync I got the following error:
> Starting Corosync Cluster Engine (corosync): /etc/init.d/corosync: line 99: 8470 Aborted $prog $COROSYNC_OPTIONS > /dev/null 2>&1
> [FAILED]
>
> if I launch corosync -f I got:
> corosync: main.c:143: logsys_qb_init: Assertion `"implicit callsite section is populated, otherwise target's build is at fault, preventing reliable logging" && __start___verbose != __stop___verbose' failed.
>
> anything is logged (even in debug mode).
>
> I do not understand why installing libqb during the normal upgrade process fails while if I upgrade it after the crmsh/pacemaker/corosync/resourceagents upgrade it works fine.
>
> On 3 Jul 2018, at 11:42, Christine Caulfield <ccaulfie at redhat.com <mailto:ccaulfie at redhat.com>> wrote:
>>
>> On 03/07/18 07:53, Jan Pokorný wrote:
>>> On 02/07/18 17:19 +0200, Salvatore D'angelo wrote:
>>>> Today I tested the two suggestions you gave me. Here what I did.
>>>> In the script where I create my 5 machines cluster (I use three
>>>> nodes for pacemaker PostgreSQL cluster and two nodes for glusterfs
>>>> that we use for database backup and WAL files).
>>>>
>>>> FIRST TEST
>>>> ——————————
>>>> I added the —shm-size=512m to the “docker create” command. I noticed
>>>> that as soon as I start it the shm size is 512m and I didn’t need to
>>>> add the entry in /etc/fstab. However, I did it anyway:
>>>>
>>>> tmpfs /dev/shm tmpfs defaults,size=512m 0 0
>>>>
>>>> and then
>>>> mount -o remount /dev/shm
>>>>
>>>> Then I uninstalled all pieces of software (crmsh, resource agents,
>>>> corosync and pacemaker) and installed the new one.
>>>> Started corosync and pacemaker but same problem occurred.
>>>>
>>>> SECOND TEST
>>>> ———————————
>>>> stopped corosync and pacemaker
>>>> uninstalled corosync
>>>> build corosync with --enable-small-memory-footprint and installed it
>>>> starte corosync and pacemaker
>>>>
>>>> IT WORKED.
>>>>
>>>> I would like to understand now why it didn’t worked in first test
>>>> and why it worked in second. Which kind of memory is used too much
>>>> here? /dev/shm seems not the problem, I allocated 512m on all three
>>>> docker images (obviously on my single Mac) and enabled the container
>>>> option as you suggested. Am I missing something here?
>>>
>>> My suspicion then fully shifts towards "maximum number of bytes of
>>> memory that may be locked into RAM" per-process resource limit as
>>> raised in one of the most recent message ...
>>>
>>>> Now I want to use Docker for the moment only for test purpose so it
>>>> could be ok to use the --enable-small-memory-footprint, but there is
>>>> something I can do to have corosync working even without this
>>>> option?
>>>
>>> ... so try running the container the already suggested way:
>>>
>>> docker run ... --ulimit memlock=33554432 ...
>>>
>>> or possibly higher (as a rule of thumb, keep doubling the accumulated
>>> value until some unreasonable amount is reached, like the equivalent
>>> of already used 512 MiB).
>>>
>>> Hope this helps.
>>
>> This makes a lot of sense to me. As Poki pointed out earlier, in
>> corosync 2.4.3 (I think) we fixed a regression in that caused corosync
>> NOT to be locked in RAM after it forked - which was causing potential
>> performance issues. So if you replace an earlier corosync with 2.4.3 or
>> later then it will use more locked memory than before.
>>
>> Chrissie
>>
>>
>>>
>>>> The reason I am asking this is that, in the future, it could be
>>>> possible we deploy in production our cluster in containerised way
>>>> (for the moment is just an idea). This will save a lot of time in
>>>> developing, maintaining and deploying our patch system. All
>>>> prerequisites and dependencies will be enclosed in container and if
>>>> IT team will do some maintenance on bare metal (i.e. install new
>>>> dependencies) it will not affects our containers. I do not see a lot
>>>> of performance drawbacks in using container. The point is to
>>>> understand if a containerised approach could save us lot of headache
>>>> about maintenance of this cluster without affect performance too
>>>> much. I am notice in Cloud environment this approach in a lot of
>>>> contexts.
>>>
>>>
>>>
>>> _______________________________________________
>>> Users mailing list: Users at clusterlabs.org <mailto:Users at clusterlabs.org>
>>> https://lists.clusterlabs.org/mailman/listinfo/users <https://lists.clusterlabs.org/mailman/listinfo/users>
>>>
>>> Project Home: http://www.clusterlabs.org <http://www.clusterlabs.org/>
>>> Getting started: http://www.clusterlabs.org/doc/Cluster_from_Scratch.pdf <http://www.clusterlabs.org/doc/Cluster_from_Scratch.pdf>
>>> Bugs: http://bugs.clusterlabs.org <http://bugs.clusterlabs.org/>
>>>
>>
>> _______________________________________________
>> Users mailing list: Users at clusterlabs.org <mailto:Users at clusterlabs.org>
>> https://lists.clusterlabs.org/mailman/listinfo/users <https://lists.clusterlabs.org/mailman/listinfo/users>
>>
>> Project Home: http://www.clusterlabs.org <http://www.clusterlabs.org/>
>> Getting started: http://www.clusterlabs.org/doc/Cluster_from_Scratch.pdf <http://www.clusterlabs.org/doc/Cluster_from_Scratch.pdf>
>> Bugs: http://bugs.clusterlabs.org <http://bugs.clusterlabs.org/>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://lists.clusterlabs.org/pipermail/users/attachments/20180706/f8086cd9/attachment-0002.html>
More information about the Users
mailing list