[Puppet Users] puppet calling stat() on managed symlinks

Hi Tim,

On Monday, 23 April 2012 05:41:31 UTC+1, Tim wrote:

[...]
[...]

This is not Puppet doing open on this directory (not explicitly anyway).
Have a look on this:

kwilczynski@desktop:~/Development/Sandbox/Test$ mkdir -p a/{b,c/d}
kwilczynski@desktop:~/Development/Sandbox/Test$ tree
.
└── a
├── b
└── c
└── d

4 directories, 0 files

kwilczynski@desktop:~/Development/Sandbox/Test$ strace -e
open,lstat,getdents ruby -e "Dir['./**/*']"
open("/etc/ld.so.cache", O_RDONLY) = 3
open("/usr/lib/libruby1.8.so.1.8", O_RDONLY) = 3
open("/lib/libpthread.so.0", O_RDONLY) = 3
open("/lib/librt.so.1", O_RDONLY) = 3
open("/lib/libdl.so.2", O_RDONLY) = 3
open("/lib/libcrypt.so.1", O_RDONLY) = 3
open("/lib/libm.so.6", O_RDONLY) = 3
open("/lib/libc.so.6", O_RDONLY) = 3
open("/dev/urandom", O_RDONLY|O_NOCTTY|O_NONBLOCK|O_NOFOLLOW) = 3
open(".", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
getdents(3, /* 3 entries */, 32768) = 72
lstat("./a", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
open("./a", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 4
getdents(4, /* 4 entries */, 32768) = 96
lstat("./a/b", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
open("./a/b", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 5
getdents(5, /* 2 entries */, 32768) = 48
getdents(5, /* 0 entries */, 32768) = 0
lstat("./a/c", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
open("./a/c", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 5
getdents(5, /* 3 entries */, 32768) = 72
lstat("./a/c/d", {st_mode=S_IFDIR|0755, st_size=0, ...}) = 0
open("./a/c/d", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 6
getdents(6, /* 2 entries */, 32768) = 48
getdents(6, /* 0 entries */, 32768) = 0
getdents(5, /* 0 entries */, 32768) = 0
getdents(4, /* 0 entries */, 32768) = 0
getdents(3, /* 0 entries */, 32768) = 0
kwilczynski@desktop:~/Development/Sandbox/Test$

And then ...

kwilczynski@desktop:~/Development/Sandbox/Test$ strace -e
open,lstat,getdents ruby -e "Dir['./*']"
open("/etc/ld.so.cache", O_RDONLY) = 3
open("/usr/lib/libruby1.8.so.1.8", O_RDONLY) = 3
open("/lib/libpthread.so.0", O_RDONLY) = 3
open("/lib/librt.so.1", O_RDONLY) = 3
open("/lib/libdl.so.2", O_RDONLY) = 3
open("/lib/libcrypt.so.1", O_RDONLY) = 3
open("/lib/libm.so.6", O_RDONLY) = 3
open("/lib/libc.so.6", O_RDONLY) = 3
open("/dev/urandom", O_RDONLY|O_NOCTTY|O_NONBLOCK|O_NOFOLLOW) = 3
open(".", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
getdents(3, /* 3 entries */, 32768) = 72
getdents(3, /* 0 entries */, 32768) = 0

Or, an equivalent:

kwilczynski@desktop:~/Development/Sandbox/Test$ strace -e
open,lstat,getdents ruby -e "Dir.entries('.')"
open("/etc/ld.so.cache", O_RDONLY) = 3
open("/usr/lib/libruby1.8.so.1.8", O_RDONLY) = 3
open("/lib/libpthread.so.0", O_RDONLY) = 3
open("/lib/librt.so.1", O_RDONLY) = 3
open("/lib/libdl.so.2", O_RDONLY) = 3
open("/lib/libcrypt.so.1", O_RDONLY) = 3
open("/lib/libm.so.6", O_RDONLY) = 3
open("/lib/libc.so.6", O_RDONLY) = 3
open("/dev/urandom", O_RDONLY|O_NOCTTY|O_NONBLOCK|O_NOFOLLOW) = 3
open(".", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
getdents(3, /* 3 entries */, 32768) = 72
getdents(3, /* 0 entries */, 32768) = 0
kwilczynski@desktop:~/Development/Sandbox/Test$

Also, two more examples:

kwilczynski@desktop:~/Development/Sandbox/Test$ cat > ftw.c
#include <ftw.h>
#include <stdio.h>
#include <stdlib.h>

int callback(const char *name, const struct stat *status, int type);

int main(int argc, char *argv[])
{
char *root = ".";

ftw((char *) root, callback, 1);

return 0;
}

int callback(const char *name, const struct stat *status, int type)
{
if (type == FTW_NS)
return 0;

if (type == FTW_F || type == FTW_SL || type == FTW_D)
printf("%s\n", name);

return 0;
}
kwilczynski@desktop:~/Development/Sandbox/Test$ gcc -o ftw ftw.c
kwilczynski@desktop:~/Development/Sandbox/Test$ ./ftw
.
./a
./a/b
./a/c
./a/c/d
./readdir.c
./readdir
./ftw.c
./ftw

kwilczynski@desktop:~/Development/Sandbox/Test$ strace -e
open,lstat,getdents ./ftw
open("/etc/ld.so.cache", O_RDONLY) = 3
open("/lib/libc.so.6", O_RDONLY) = 3
open(".", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
.
getdents(3, /* 7 entries */, 32768) = 192
getdents(3, /* 0 entries */, 32768) = 0
open("./a", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
./a
getdents(3, /* 4 entries */, 32768) = 96
getdents(3, /* 0 entries */, 32768) = 0
open("./a/b", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
./a/b
getdents(3, /* 2 entries */, 32768) = 48
getdents(3, /* 0 entries */, 32768) = 0
open("./a/c", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
./a/c
getdents(3, /* 3 entries */, 32768) = 72
getdents(3, /* 0 entries */, 32768) = 0
open("./a/c/d", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
./a/c/d
getdents(3, /* 2 entries */, 32768) = 48
getdents(3, /* 0 entries */, 32768) = 0
./readdir.c
./readdir
./ftw.c
./ftw
kwilczynski@desktop:~/Development/Sandbox/Test$

And a non-recursive one:

kwilczynski@desktop:~/Development/Sandbox/Test$ cat > readdir.c
#include <stdio.h>
#include <stdlib.h>
#include <dirent.h>

struct dirent *d_ent;

int main(int argc, char *argv[])
{
char *root = ".";

DIR *d_ptr;

if((d_ptr = opendir((char *) root)) == NULL) {
printf("Error");
exit(1);
}

while(d_ent = readdir(d_ptr))
printf("%s\n", d_ent->d_name);

closedir(d_ptr);

return 0;
}
kwilczynski@desktop:~/Development/Sandbox/Test$ gcc -o readdir readdir.c
kwilczynski@desktop:~/Development/Sandbox/Test$ ./readdir
.
..
a
readdir.c
readdir

kwilczynski@desktop:~/Development/Sandbox/Test$ strace -e
open,lstat,getdents ./readdir
open("/etc/ld.so.cache", O_RDONLY) = 3
open("/lib/libc.so.6", O_RDONLY) = 3
open(".", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 3
getdents(3, /* 5 entries */, 32768) = 136
.
..
a
readdir.c
readdir
getdents(3, /* 0 entries */, 32768) = 0
kwilczynski@desktop:~/Development/Sandbox/Test$

Can you see what is going on when you request recursive directory
traversal? :)

You'd have to probably remove "recurse => true" from your File resource
that is looking after a directory structure in this case.

Therefore, your open() is a needed to get the file hand (file descriptor)
on a new directory of interest when traversing. Almost everything is a file
in Unix / Linux :)

KW

[...]

I forgot to add ...

You probably do not want Puppet to follow symbolic links which in
conjunction with the "recurse => true" is leading to open() done on each
new directory during traversal. Perhaps setting "links => ignore" will
help, not sure, thought.

KW

That sounds sensible, but I don't know whether it's practical; some of
Puppet's behaviors are constrained by the underlying Ruby libraries.
In any case, the first thing to try is to set "links => manage" on
your File resource (not "links => ignore", which cannot work when the
desired links are not yet present).

The bottom line, however, is that recursive File management has always
been a sore spot for Puppet, and it probably always will be. It is
not remotely possible to provide enough information in the parameters
of a single File resource to cover all the possible permutations of
desired behavior. Puppet does a fairly good job of coping with that
when asked to do so, but generally you are better off finding a
different approach.

If you have a large number of files, then you may be better off
wrapping the whole thing into a custom package (e.g. an RPM), putting
that in a local repository or on a network filesystem, and managing
that package instead of the individual files. If there are only a few
files (that you want to manage) then by all means set up individual
file resources for them.


John

I've ended up doing as John said and creating file resources for all
of these directories and links instead of syncing from an existing
source.

Works a treat :) Thanks for your help all.

Cheers,
Tim