I ran strace on splunk start to see if I could see what it was doing but don't see anything in particular.
09:30:40.542721 ioctl(3, SNDCTL_TMR_START or SNDRV_TIMER_IOCTL_TREAD or TCSETS, {B38400 opost isig icanon echo ...}) = 0
09:30:40.542750 ioctl(3, SNDCTL_TMR_TIMEBASE or SNDRV_TIMER_IOCTL_NEXT_DEVICE or TCGETS, {B38400 opost isig icanon echo ...}) = 0
09:30:40.542767 close(3) = 0
09:30:40.542793 pipe2([3, 4], O_CLOEXEC) = 0
09:30:40.542832 clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7fb4024d99d0) = 19725
09:30:40.542942 close(4) = 0
09:30:40.542958 fcntl(3, F_SETFD, 0) = 0
09:30:40.542978 fstat(3, {st_mode=S_IFIFO|0600, st_size=0, ...}) = 0
09:30:40.542997 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fb4024f1000
09:30:40.543015 read(3, "[clustering]\naccess_logging_for_"..., 4096) = 1224
09:30:40.561740 read(3, "", 4096) = 0
09:30:40.562243 close(3) = 0
09:30:40.562263 wait4(19725, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0, NULL) = 19725
09:30:40.562294 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=19725, si_status=0, si_utime=1, si_stime=0} ---
09:30:40.562306 munmap(0x7fb4024f1000, 4096) = 0
09:30:40.562338 open("/etc/localtime", O_RDONLY) = 3
09:30:40.562390 fstat(3, {st_mode=S_IFREG|0644, st_size=2819, ...}) = 0
09:30:40.562412 fstat(3, {st_mode=S_IFREG|0644, st_size=2819, ...}) = 0
09:30:40.562426 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fb4024f1000
09:30:40.562443 read(3, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\4\0\0\0\4\0\0\0\0"..., 4096) = 2819
09:30:40.562461 lseek(3, -1802, SEEK_CUR) = 1017
09:30:40.562476 read(3, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\5\0\0\0\5\0\0\0\0"..., 4096) = 1802
09:30:40.562496 close(3) = 0
09:30:40.562510 munmap(0x7fb4024f1000, 4096) = 0
09:30:40.562536 open("/apps/splunk/var/log/splunk/migration.log.2016-04-08.09-30-40", O_WRONLY|O_CREAT|O_APPEND, 0666) = 3
09:30:40.562717 fstat(3, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0
09:30:40.562740 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fb4024f1000
09:30:40.562757 fstat(3, {st_mode=S_IFREG|0600, st_size=0, ...}) = 0
09:30:40.562771 lseek(3, 0, SEEK_SET) = 0
09:30:40.562802 write(2, "\n-- Migration information is bei"..., 112
-- Migration information is being logged to '/apps/splunk/var/log/splunk/migration.log.2016-04-08.09-30-40' --
) = 112
09:30:40.562824 write(1, "\nMigrating to:\n", 15
Migrating to:
) = 15
09:30:40.562853 open("/apps/splunk/etc/splunk.version", O_RDONLY) = 4
09:30:40.562877 fstat(4, {st_mode=S_IFREG|0755, st_size=70, ...}) = 0
09:30:40.562892 mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fb4024f0000
09:30:40.562908 read(4, "VERSION=6.4.0\nBUILD=f2c836328108"..., 4096) = 70
09:30:40.562925 read(4, "", 4096) = 0
09:30:40.562939 close(4) = 0
09:30:40.562952 munmap(0x7fb4024f0000, 4096) = 0
09:30:40.562972 write(1, "VERSION=6.4.0\nBUILD=f2c836328108"..., 70VERSION=6.4.0
BUILD=f2c836328108
PRODUCT=splunk
PLATFORM=Linux-x86_64
) = 70
09:30:40.562989 write(1, "\n", 1
) = 1
09:30:40.563005 write(3, "\nMigrating to:\nVERSION=6.4.0\nBUI"..., 86) = 86
09:30:40.563031 close(3) = 0
09:30:40.563044 munmap(0x7fb4024f1000, 4096) = 0
09:30:40.563063 clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x7fb4024d99d0) = 19726
09:30:40.563143 wait4(19726, Can't create directory "": No such file or directory
An error occurred: Could not create audit keys (returned 4).
[{WIFEXITED(s) && WEXITSTATUS(s) == 2}], 0, NULL) = 19726
09:30:40.729452 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=19726, si_status=2, si_utime=7, si_stime=2} ---
09:30:40.729511 exit_group(2) = ?
09:30:40.729575 +++ exited with 2 +++
[root@splunk-id1 bin]#
... View more