[ABRT PATCH 4/5] spec: add abrt-upload-watch

[Jakub Filak]

Related to #657

Signed-off-by: Jakub Filak <jfilak-H+wXaHxf7aLQT0dZR+***@public.gmane.org>
---
abrt.spec.in | 29 +++++++++++++++++++++++++++++
1 file changed, 29 insertions(+)

diff --git a/abrt.spec.in b/abrt.spec.in
index 5f646e5..b459f24 100644
--- a/abrt.spec.in
+++ b/abrt.spec.in
@@ -150,6 +150,15 @@ Requires: abrt-libs = %{version}-%{release}
This package contains hook for C/C++ crashed programs and %{name}'s C/C++
analyzer plugin.

+%package addon-upload-watch
+Summary: %{name}'s upload addon
+Group: System Environment/Libraries
+Requires: %{name} = %{version}-%{release}
+Requires: abrt-libs = %{version}-%{release}
+
+%description addon-upload-watch
+This package contains hook for uploaded problems.
+
%package retrace-client
Summary: %{name}'s retrace client
Group: System Environment/Libraries
@@ -437,6 +446,9 @@ chown -R abrt:abrt %{_localstatedir}/cache/abrt-di
%post addon-uefioops
%systemd_post abrt-uefioops.service

+%post addon-upload-watch
+%systemd_post abrt-upload-watch.service
+
%preun
%systemd_preun abrtd.service

@@ -455,6 +467,9 @@ chown -R abrt:abrt %{_localstatedir}/cache/abrt-di
%preun addon-uefioops
%systemd_preun abrt-uefioops.service

+%preun addon-upload-watch
+%systemd_preun abrt-upload-watch.service
+
%postun
%systemd_postun_with_restart abrtd.service

@@ -473,6 +488,9 @@ chown -R abrt:abrt %{_localstatedir}/cache/abrt-di
%postun addon-uefioops
%systemd_postun_with_restart abrt-uefioops.service

+%postun addon-upload-watch
+%systemd_postun_with_restart abrt-upload-watch.service
+
%post gui
# update icon cache
touch --no-create %{_datadir}/icons/hicolor &>/dev/null || :
@@ -646,6 +664,17 @@ gtk-update-icon-cache %{_datadir}/icons/hicolor &>/dev/null || :
%{_mandir}/man*/abrt-action-perform-ccpp-analysis.*
%{_mandir}/man*/abrt-dedup-client.*

+%files addon-upload-watch
+%defattr(-,root,root,-)
+%{_sbindir}/abrt-upload-watch
+%if %{with systemd}
+%{_unitdir}/abrt-upload-watch.service
+%else
+%{_initrddir}/abrt-upload-watch
+%endif
+%{_mandir}/man*/abrt-upload-watch.*
+
+
%files retrace-client
%{_bindir}/abrt-retrace-client
%{_mandir}/man1/abrt-retrace-client.1.gz

[Jakub Filak]

Related to #657

Signed-off-by: Jakub Filak <jfilak-H+wXaHxf7aLQT0dZR+***@public.gmane.org>
---
Makefile.am | 6 ++-
init-scripts/abrt-upload-watch | 93 ++++++++++++++++++++++++++++++++++
init-scripts/abrt-upload-watch.service | 10 ++++
3 files changed, 107 insertions(+), 2 deletions(-)
create mode 100644 init-scripts/abrt-upload-watch
create mode 100644 init-scripts/abrt-upload-watch.service

diff --git a/Makefile.am b/Makefile.am
index b56a954..a67659e 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -40,7 +40,8 @@ if HAVE_SYSTEMD
init-scripts/abrt-oops.service \
init-scripts/abrt-xorg.service \
init-scripts/abrt-vmcore.service \
- init-scripts/abrt-uefioops.service
+ init-scripts/abrt-uefioops.service \
+ init-scripts/abrt-upload-watch.service
else
sysv_initdir = $(sysconfdir)/rc.d/init.d/
sysv_init_SCRIPTS = init-scripts/abrtd \
@@ -48,7 +49,8 @@ else
init-scripts/abrt-oops \
init-scripts/abrt-xorg \
init-scripts/abrt-vmcore \
- init-scripts/abrt-uefioops
+ init-scripts/abrt-uefioops \
+ init-scripts/abrt-upload-watch
endif

RPM_DIRS = --define "_sourcedir `pwd`" \
diff --git a/init-scripts/abrt-upload-watch b/init-scripts/abrt-upload-watch
new file mode 100644
index 0000000..f1d6a04
--- /dev/null
+++ b/init-scripts/abrt-upload-watch
@@ -0,0 +1,93 @@
+#!/bin/bash
+# Watches upload directory and extracts incoming archives into ABRT dump
+# location
+#
+# chkconfig: 35 82 16
+# description: ABRT upload watch
+### BEGIN INIT INFO
+# Provides: abrt-upload-watch
+# Required-Start: $abrtd
+# Default-Stop: 0 1 2 6
+# Default-Start: 3 5
+# Short-Description: ABRT upload watch
+# Description: Extracts incoming ABRT archives into ABRT dump location
+### END INIT INFO
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+PROG="abrt-upload-watch"
+LOCKFILE="/var/lock/subsys/$PROG"
+EXEC="/usr/sbin/$PROG"
+
+start() {
+ [ -x $EXEC ] || exit 5
+ echo -n $"Starting '$PROG': "
+ daemon $EXEC -d
+ RETVAL=$?
+ echo
+ [ $RETVAL -eq 0 ] && touch $LOCKFILE
+ return $RETVAL
+}
+
+stop() {
+ echo -n $"Stopping '$PROG': "
+ killproc $PROG
+ RETVAL=$?
+ echo
+ [ $RETVAL -eq 0 ] && rm -f $LOCKFILE
+ return $RETVAL
+}
+
+restart() {
+ stop
+ start
+}
+
+reload() {
+ restart
+}
+
+force_reload() {
+ restart
+}
+
+rh_status() {
+ status $PROG
+}
+
+rh_status_q() {
+ rh_status >/dev/null 2>&1
+}
+
+case "$1" in
+ start)
+ rh_status_q && exit 0
+ $1
+ ;;
+ stop)
+ rh_status_q || exit 0
+ $1
+ ;;
+ restart)
+ $1
+ ;;
+ reload)
+ rh_status_q || exit 7
+ $1
+ ;;
+ force-reload)
+ force_reload
+ ;;
+ status)
+ rh_status
+ ;;
+ condrestart|try-restart)
+ rh_status_q || exit 0
+ restart
+ ;;
+ *)
+ echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
+ exit 2
+esac
+exit $?
diff --git a/init-scripts/abrt-upload-watch.service b/init-scripts/abrt-upload-watch.service
new file mode 100644
index 0000000..47ccf71
--- /dev/null
+++ b/init-scripts/abrt-upload-watch.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=ABRT upload watcher
+After=abrtd.service
+Requisite=abrtd.service
+
+[Service]
+ExecStart=/usr/sbin/abrt-upload-watch
+
+[Install]
+WantedBy=multi-user.target

[Jakub Filak]

Related to #657

Signed-off-by: Jakub Filak <jfilak-H+wXaHxf7aLQT0dZR+***@public.gmane.org>
---
doc/Makefile.am | 1 +
doc/abrt-upload-watch.txt | 51 ++++++
po/POTFILES.in | 1 +
src/daemon/Makefile.am | 25 ++-
src/daemon/abrt-inotify.c | 175 +++++++++++++++++++
src/daemon/abrt-inotify.h | 40 +++++
src/daemon/abrt-upload-watch.c | 378 +++++++++++++++++++++++++++++++++++++++++
src/daemon/abrtd.c | 175 ++-----------------
8 files changed, 680 insertions(+), 166 deletions(-)
create mode 100644 doc/abrt-upload-watch.txt
create mode 100644 src/daemon/abrt-inotify.c
create mode 100644 src/daemon/abrt-inotify.h
create mode 100644 src/daemon/abrt-upload-watch.c

diff --git a/doc/Makefile.am b/doc/Makefile.am
index 872d210..e0c50ff 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -33,6 +33,7 @@ MAN1_TXT += abrt-install-ccpp-hook.txt
MAN1_TXT += abrt-action-analyze-vmcore.txt
MAN1_TXT += abrt-action-analyze-ccpp-local.txt
MAN1_TXT += abrt-watch-log.txt
+MAN1_TXT += abrt-upload-watch.txt
MAN1_TXT += system-config-abrt.txt
if BUILD_BODHI
MAN1_TXT += abrt-bodhi.txt
diff --git a/doc/abrt-upload-watch.txt b/doc/abrt-upload-watch.txt
new file mode 100644
index 0000000..1486777
--- /dev/null
+++ b/doc/abrt-upload-watch.txt
@@ -0,0 +1,51 @@
+abrt-upload-watch(1)
+==================
+
+NAME
+----
+abrt-upload-watch - Watch upload directory and unpacks incoming archives into DumpLocation
+
+SYNOPSIS
+--------
+'abrt-upload-watch' [-vs] [-w NUM_WORKERS] [-c CACHE_SIZE_MIB] [UPLOAD_DIRECTORY]
+
+OPTIONS
+-------
+-v, --verbose::
+ Be more verbose. Can be given multiple times.
+
+-s::
+ Log to syslog
+
+-d::
+ Daemonize
+
+-w NUM_WORKERS::
+ Number of concurrent workers. Default is 10
+
+-c CACHE_SIZE_MIB::
+ Maximal cache size in MiB. Default is 4
+
+UPLOAD_DIRECTORY::
+ Watched directory. Default is a value of WatchCrashdumpArchiveDir option from abrt.conf
+
+FILES
+-----
+Uses these three configuration options from file '/etc/abrt/abrt.conf':
+
+WatchCrashdumpArchiveDir::
+ Default upload directory
+
+DumpLocation::
+ Place where uploaded archives are unpacked
+
+DeleteUploaded::
+ Specifies if uploaded archives are deleted after unpacking
+
+SEE ALSO
+--------
+abrt.conf(5)
+
+AUTHORS
+-------
+* ABRT team
diff --git a/po/POTFILES.in b/po/POTFILES.in
index 5e14bc8..8d22e35 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -11,6 +11,7 @@ src/daemon/abrt-server.c
src/dbus/abrt-dbus.c
src/daemon/abrtd.c
src/daemon/abrt-handle-event.c
+src/daemon/abrt-upload-watch.c
src/lib/abrt_conf.c
src/lib/hooklib.c
src/lib/problem_api.c
diff --git a/src/daemon/Makefile.am b/src/daemon/Makefile.am
index 33eec63..9369d26 100644
--- a/src/daemon/Makefile.am
+++ b/src/daemon/Makefile.am
@@ -11,14 +11,17 @@ bin_PROGRAMS = \

sbin_PROGRAMS = \
abrtd \
- abrt-server
+ abrt-server \
+ abrt-upload-watch

libexec_PROGRAMS = abrt-handle-event

# This is a daemon, building with full relro and PIE
# for increased security.
abrtd_SOURCES = \
- abrtd.c
+ abrtd.c \
+ abrt-inotify.c \
+ abrt-inotify.h
abrtd_CPPFLAGS = \
-I$(srcdir)/../include \
-I$(srcdir)/../lib \
@@ -49,6 +52,24 @@ abrt_server_LDADD = \
../lib/libabrt.la \
$(LIBREPORT_LIBS)

+abrt_upload_watch_SOURCES = \
+ abrt-upload-watch.c \
+ abrt-inotify.c \
+ abrt-inotify.h
+abrt_upload_watch_CPPFLAGS = \
+ -I$(srcdir)/../include \
+ -I$(srcdir)/../lib \
+ -DDEFAULT_DUMP_DIR_MODE=$(DEFAULT_DUMP_DIR_MODE) \
+ -DLIBEXEC_DIR=\"$(libexecdir)\" \
+ $(GLIB_CFLAGS) \
+ $(GIO_CFLAGS) \
+ $(LIBREPORT_CFLAGS) \
+ -D_GNU_SOURCE
+abrt_upload_watch_LDADD = \
+ ../lib/libabrt.la \
+ $(LIBREPORT_LIBS)
+
+
abrt_handle_event_SOURCES = \
abrt-handle-event.c
abrt_handle_event_CPPFLAGS = \
diff --git a/src/daemon/abrt-inotify.c b/src/daemon/abrt-inotify.c
new file mode 100644
index 0000000..ffc5fd0
--- /dev/null
+++ b/src/daemon/abrt-inotify.c
@@ -0,0 +1,175 @@
+/*
+ Copyright (C) 2013 RedHat inc.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include "abrt-inotify.h"
+#include "abrt_glib.h"
+#include "libabrt.h"
+
+#include <stdio.h>
+#include <sys/ioctl.h> /* ioctl(FIONREAD) */
+
+struct abrt_inotify_watch
+{
+ abrt_inotify_watch_handler handler;
+ void *user_data;
+ int inotify_fd;
+ int inotify_wd;
+ GIOChannel *channel_inotify;
+ guint channel_inotify_source_id;
+};
+
+/* Inotify handler */
+
+static gboolean handle_inotify_cb(GIOChannel *gio, GIOCondition condition, gpointer user_data)
+{
+ /* Default size: 128 simultaneous actions (about 1/2 meg) */
+#define INOTIFY_BUF_SIZE ((sizeof(struct inotify_event) + FILENAME_MAX)*128)
+ /* Determine how much to read (it usually is much smaller) */
+ /* NB: this variable _must_ be int-sized, ioctl expects that! */
+ int inotify_bytes = INOTIFY_BUF_SIZE;
+ if (ioctl(g_io_channel_unix_get_fd(gio), FIONREAD, &inotify_bytes) != 0
+ /*|| inotify_bytes < sizeof(struct inotify_event)
+ ^^^^^^^^^^^^^^^^^^^ - WRONG: legitimate 0 was seen when flooded with inotify events
+ */
+ || inotify_bytes > INOTIFY_BUF_SIZE
+ ) {
+ inotify_bytes = INOTIFY_BUF_SIZE;
+ }
+ VERB3 log("FIONREAD:%d", inotify_bytes);
+
+ if (inotify_bytes == 0)
+ return TRUE; /* "please don't remove this event" */
+
+ /* We may race: more inotify events may happen after ioctl(FIONREAD).
+ * To be more efficient, allocate a bit more space to eat those events too.
+ * This also would help against a bug we once had where
+ * g_io_channel_read_chars() was buffering reads
+ * and we were going out of sync wrt struct inotify_event's layout.
+ */
+ inotify_bytes += 2 * (sizeof(struct inotify_event) + FILENAME_MAX);
+ char *buf = xmalloc(inotify_bytes);
+ errno = 0;
+ gsize len;
+ GError *gerror = NULL;
+ /* Note: we ensured elsewhere that this read is non-blocking, making it ok
+ * for buffer len (inotify_bytes) to be larger than actual available byte count.
+ */
+ GIOStatus err = g_io_channel_read_chars(gio, buf, inotify_bytes, &len, &gerror);
+ if (err != G_IO_STATUS_NORMAL)
+ {
+ perror_msg("Error reading inotify fd: %s", gerror ? gerror->message : "unknown");
+ free(buf);
+ if (gerror)
+ g_error_free(gerror);
+ return FALSE; /* "remove this event" (huh??) */
+ }
+
+ struct abrt_inotify_watch *aic = (struct abrt_inotify_watch *)user_data;
+ /* Reconstruct each event */
+ gsize i = 0;
+ for (;;)
+ {
+ if (i >= len)
+ {
+ /* This would catch one of our former bugs. Let's be paranoid */
+ if (i > len)
+ error_msg("warning: ran off struct inotify (this should never happen): %u > %u", (int)i, (int)len);
+ break;
+ }
+ struct inotify_event *event = (struct inotify_event *) &buf[i];
+ i += sizeof(*event) + event->len;
+
+ aic->handler(aic, event, aic->user_data);
+ }
+ free(buf);
+ return TRUE;
+}
+
+struct abrt_inotify_watch *
+abrt_inotify_watch_init(const char *path, int inotify_flags, abrt_inotify_watch_handler handler, void *user_data)
+{
+ struct abrt_inotify_watch *aiw = malloc(sizeof(*aiw));
+ aiw->handler = handler;
+ aiw->user_data = user_data;
+
+ VERB1 log("Initializing inotify");
+ errno = 0;
+ aiw->inotify_fd = inotify_init();
+ if (aiw->inotify_fd == -1)
+ perror_msg_and_die("inotify_init failed");
+ close_on_exec_on(aiw->inotify_fd);
+
+ aiw->inotify_wd = inotify_add_watch(aiw->inotify_fd, path, inotify_flags);
+ if (aiw->inotify_wd < 0)
+ perror_msg_and_die("inotify_add_watch failed on '%s'", path);
+
+ VERB1 log("Adding inotify watch to glib main loop");
+ /* Without nonblocking mode, users observed abrtd blocking
+ * on inotify read forever. Must set fd to non-blocking:
+ */
+ ndelay_on(aiw->inotify_fd);
+ aiw->channel_inotify = abrt_gio_channel_unix_new(aiw->inotify_fd);
+
+ /*
+ * glib's read buffering must be disabled, or else
+ * FIONREAD-reported "available data" sizes and sizes of reads
+ * can become inconsistent, and worse, buffering can split
+ * struct inotify's (very bad!).
+ */
+ g_io_channel_set_buffered(aiw->channel_inotify, false);
+
+ errno = 0;
+ aiw->channel_inotify_source_id = g_io_add_watch(aiw->channel_inotify,
+ G_IO_IN | G_IO_PRI | G_IO_HUP,
+ handle_inotify_cb,
+ aiw);
+ if (!aiw->channel_inotify_source_id)
+ perror_msg_and_die("g_io_add_watch failed");
+
+ return aiw;
+}
+
+void
+abrt_inotify_watch_reset(struct abrt_inotify_watch *watch, const char *path, int inotify_flags)
+{
+ inotify_rm_watch(watch->inotify_fd, watch->inotify_wd);
+ watch->inotify_wd = inotify_add_watch(watch->inotify_fd, path, inotify_flags);
+ if (watch->inotify_wd < 0)
+ perror_msg_and_die("inotify_add_watch failed on '%s'", path);
+}
+
+void
+abrt_inotify_watch_destroy(struct abrt_inotify_watch *watch)
+{
+ if (!watch)
+ return;
+
+ inotify_rm_watch(watch->inotify_fd, watch->inotify_wd);
+ g_source_remove(watch->channel_inotify_source_id);
+
+ GError *error = NULL;
+ g_io_channel_shutdown(watch->channel_inotify, FALSE, &error);
+ if (error)
+ {
+ VERB1 log("Can't shutdown inotify gio channel: '%s'", error ? error->message : "");
+ g_error_free(error);
+ }
+
+ g_io_channel_unref(watch->channel_inotify);
+ free(watch);
+}
diff --git a/src/daemon/abrt-inotify.h b/src/daemon/abrt-inotify.h
new file mode 100644
index 0000000..7674fc0
--- /dev/null
+++ b/src/daemon/abrt-inotify.h
@@ -0,0 +1,40 @@
+/*
+ Copyright (C) 2013 RedHat inc.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef _ABRT_INOTIFY_H_
+#define _ABRT_INOTIFY_H_
+
+#include <sys/inotify.h>
+
+struct abrt_inotify_watch;
+
+typedef void (* abrt_inotify_watch_handler)(
+ struct abrt_inotify_watch *watch,
+ struct inotify_event *event,
+ void *user_data);
+
+struct abrt_inotify_watch *
+abrt_inotify_watch_init(const char *path, int inotify_flags, abrt_inotify_watch_handler handler, void *user_data);
+
+void
+abrt_inotify_watch_destroy(struct abrt_inotify_watch *watch);
+
+void
+abrt_inotify_watch_reset(struct abrt_inotify_watch *watch, const char *path, int inotify_flags);
+
+#endif /*_ABRT_INOTIFY_H_*/
diff --git a/src/daemon/abrt-upload-watch.c b/src/daemon/abrt-upload-watch.c
new file mode 100644
index 0000000..400a456
--- /dev/null
+++ b/src/daemon/abrt-upload-watch.c
@@ -0,0 +1,378 @@
+/*
+ Copyright (C) 2013 ABRT Team
+ Copyright (C) 2013 Red Hat, Inc.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+#include "abrt-inotify.h"
+#include "abrt_glib.h"
+#include "libabrt.h"
+
+#include <syslog.h>
+
+#define STRINGIZE_DETAIL(str) #str
+#define STRINGIZE(str) STRINGIZE_DETAIL(str)
+
+#define DEFAULT_COUNT_OF_WORKERS 10
+#define DEFAULT_CACHE_MIB_SIZE 4
+#define APPROX_BYTES_PER_ARCHIVE_NAME FILENAME_MAX
+
+static int g_signal_pipe[2];
+
+struct queue
+{
+ unsigned capacity;
+ GQueue q;
+};
+
+static int
+queue_push(struct queue *queue, char *value)
+{
+ if (g_queue_get_length(&queue->q) >= queue->capacity)
+ return 0;
+
+ g_queue_push_head(&queue->q, value);
+
+ return 1;
+}
+
+static char *
+queue_pop(struct queue *queue)
+{
+ if (g_queue_is_empty(&queue->q))
+ return NULL;
+
+ return (char *)g_queue_pop_tail(&queue->q);
+}
+
+struct process
+{
+ GMainLoop *main_loop;
+ const char *upload_directory;
+ unsigned children;
+ unsigned max_children;
+ struct queue queue;
+};
+
+static void
+process_quit(struct process *proc)
+{
+ g_main_loop_quit(proc->main_loop);
+}
+
+static void
+run_abrt_handle_upload(struct process *proc, const char *name)
+{
+ VERB2 log("Processing file '%s' in directory '%s'", name, proc->upload_directory);
+
+ ++proc->children;
+ VERB3 log("Running workers: %d", proc->children);
+
+ fflush(NULL); /* paranoia */
+ pid_t pid = fork();
+ if (pid < 0)
+ {
+ --proc->children;
+ perror_msg("fork");
+ return;
+ }
+
+ if (pid == 0)
+ {
+ /* child */
+ xchdir(proc->upload_directory);
+ if (g_settings_delete_uploaded)
+ execlp("abrt-handle-upload", "abrt-handle-upload", "-d",
+ g_settings_dump_location, proc->upload_directory, name, (char*)NULL);
+ else
+ execlp("abrt-handle-upload", "abrt-handle-upload",
+ g_settings_dump_location, proc->upload_directory, name, (char*)NULL);
+ error_msg_and_die("Can't execute '%s'", "abrt-handle-upload");
+ }
+}
+
+static void
+handle_new_path(struct process *proc, char *name)
+{
+ log("Detected creation of file '%s' in upload directory '%s'", name, proc->upload_directory);
+
+ if (proc->children < proc->max_children)
+ {
+ run_abrt_handle_upload(proc, name);
+ free(name);
+ return;
+ }
+
+ VERB3 log("Pushing '%s' to deferred queue", name);
+ if (!queue_push(&proc->queue, name))
+ {
+ error_msg(_("No free workers and full buffer. Omitting archive '%s'"), name);
+ free(name);
+ return;
+ }
+}
+
+static void
+decrement_child_count(struct process *proc)
+{
+ --proc->children;
+
+ char *name = queue_pop(&proc->queue);
+ if (!name)
+ {
+ VERB3 log("Deferred queue is empty. Running workers: %d", proc->children);
+ return;
+ }
+
+ run_abrt_handle_upload(proc, name);
+ free(name);
+}
+
+static void
+handle_signal(int signo)
+{
+ int save_errno = errno;
+ uint8_t sig_caught = signo;
+ if (write(g_signal_pipe[1], &sig_caught, 1))
+ /* we ignore result, if () shuts up stupid compiler */;
+ errno = save_errno;
+}
+
+static gboolean
+handle_sigchld_pipe(GIOChannel *gio, GIOCondition condition, gpointer user_data)
+{
+ struct process *proc = (struct process *)user_data;
+ uint8_t signals[DEFAULT_COUNT_OF_WORKERS];
+ gsize len = 0;
+
+ for (;;)
+ {
+ GError *error = NULL;
+ GIOStatus stat = g_io_channel_read_chars(gio, (void *)signals, sizeof(signals), &len, NULL);
+ if (stat == G_IO_STATUS_ERROR)
+ {
+ error_msg_and_die(_("Can't read from gio channel: '%s'"), error ? error->message : "");
+ }
+ if (stat == G_IO_STATUS_AGAIN)
+ { /* We got all buffered data, but fd is still open. Done for now */
+ return TRUE; /* "glib, please don't remove this event (yet)" */
+ }
+ if (stat == G_IO_STATUS_EOF)
+ break;
+
+ /* G_IO_STATUS_NORMAL */
+ for (unsigned signo = 0; signo < len; ++signo)
+ {
+ /* we did receive a signal */
+ VERB3 log("Got signal %d through signal pipe", signals[signo]);
+ if (signals[signo]!= SIGCHLD)
+ {
+ process_quit(proc);
+ return FALSE; /* remove this event */
+ }
+ else
+ {
+ while (safe_waitpid(-1, NULL, WNOHANG) > 0)
+ {
+ decrement_child_count(proc);
+ }
+ }
+ }
+ }
+
+ return TRUE; /* "please don't remove this event" */
+}
+
+static void
+handle_inotify_cb(struct abrt_inotify_watch *watch, struct inotify_event *event, void *user_data)
+{
+ /* Was the (presumable newly created) file closed in upload dir,
+ * or a file moved to upload dir? */
+ if (event->name && !(event->mask & IN_ISDIR) && (event->mask & (IN_CLOSE_WRITE | IN_MOVED_TO)))
+ {
+ const char *ext = strrchr(event->name, '.');
+ if (ext && strcmp(ext + 1, "working") == 0)
+ return;
+
+ handle_new_path((struct process *)user_data, xstrdup(event->name));
+ }
+}
+
+static void
+daemonize()
+{
+ /* forking to background */
+ fflush(NULL); /* paranoia */
+ pid_t pid = fork();
+ if (pid < 0)
+ perror_msg_and_die("fork");
+ if (pid > 0)
+ exit(0);
+
+ /* Child (daemon) continues */
+ if (setsid() < 0)
+ perror_msg_and_die("setsid");
+
+ /* Change the current working directory */
+ if ((chdir("/")) < 0)
+ perror_msg_and_die("chdir(/)");
+
+ /* Close out the standard file descriptors */
+ close(STDIN_FILENO);
+ close(STDOUT_FILENO);
+ close(STDERR_FILENO);
+}
+
+int
+main(int argc, char **argv)
+{
+ /* I18n */
+ setlocale(LC_ALL, "");
+#if ENABLE_NLS
+ bindtextdomain(PACKAGE, LOCALEDIR);
+ textdomain(PACKAGE);
+#endif
+
+ abrt_init(argv);
+ /* Can't keep these strings/structs static: _() doesn't support that */
+ const char *program_usage_string = _(
+ "& [-vs] [-w NUM] [-c MiB] [UPLOAD_DIRECTORY]\n"
+ "\n"
+ "\nWatches UPLOAD_DIRECTORY and unpacks incoming archives into DumpLocation"
+ "\nspecified in abrt.conf"
+ "\n"
+ "\nIf UPLOAD_DIRECTORY is not provided, uses a value of"
+ "\nWatchCrashdumpArchiveDir option from abrt.conf"
+ );
+ enum {
+ OPT_v = 1 << 0,
+ OPT_s = 1 << 1,
+ OPT_d = 1 << 2,
+ OPT_w = 1 << 3,
+ OPT_c = 1 << 4,
+ };
+
+ int concurrent_workers = DEFAULT_COUNT_OF_WORKERS;
+ int cache_size_mib = DEFAULT_CACHE_MIB_SIZE;
+
+ /* Keep enum above and order of options below in sync! */
+ struct options program_options[] = {
+ OPT__VERBOSE(&g_verbose),
+ OPT_BOOL('s', NULL, NULL , _("Log to syslog")),
+ OPT_BOOL('d', NULL, NULL , _("Daemize")),
+ OPT_INTEGER('w', NULL, &concurrent_workers, _("Number of concurrent workers. Default is "STRINGIZE(DEFAULT_COUNT_OF_WORKERS))),
+ OPT_INTEGER('c', NULL, &cache_size_mib, _("Maximal cache size in MiB. Default is "STRINGIZE(DEFAULT_CACHE_MIB_SIZE))),
+ OPT_END()
+ };
+ unsigned opts = parse_opts(argc, argv, program_options, program_usage_string);
+
+ if (concurrent_workers <= 0)
+ perror_msg_and_die("Invalid number of workers: %d", concurrent_workers);
+
+ if (cache_size_mib <= 0)
+ perror_msg_and_die("Invalid cache size in MiB: %d", cache_size_mib);
+
+ if (cache_size_mib > UINT_MAX / (1024 * 1024 / FILENAME_MAX))
+ perror_msg_and_die("Too big cache size. Maximum is : %u MiB", UINT_MAX / (1024 * 1024 / FILENAME_MAX));
+
+ struct process proc = {0};
+ proc.max_children = concurrent_workers;
+ /* By default it is about 1024 entries */
+ g_queue_init(&proc.queue.q);
+ proc.queue.capacity = cache_size_mib * (1024 * 1024 / FILENAME_MAX);
+ VERB3 log("Max queue size %u", proc.queue.capacity);
+
+ argv += optind;
+ if (argv[0])
+ {
+ proc.upload_directory = argv[0];
+
+ if (argv[1])
+ show_usage_and_die(program_usage_string, program_options);
+ }
+
+ /* Initialization */
+ VERB2 log("Loading settings");
+ if (load_abrt_conf() != 0)
+ return 1;
+
+ if (!proc.upload_directory)
+ proc.upload_directory = g_settings_sWatchCrashdumpArchiveDir;
+
+ if (!proc.upload_directory)
+ error_msg_and_die("Neither UPLOAD_DIRECTORY nor WatchCrashdumpArchiveDir was specified");
+
+ if (opts & OPT_d)
+ daemonize();
+
+ msg_prefix = g_progname;
+ if ((opts & OPT_d) || (opts & OPT_s) || getenv("ABRT_SYSLOG"))
+ {
+ openlog(msg_prefix, 0, LOG_DAEMON);
+ logmode = LOGMODE_SYSLOG;
+ }
+
+ VERB2 log("Creating glib main loop");
+ proc.main_loop = g_main_loop_new(NULL, FALSE);
+
+ VERB1 log("Setting up a file monitor for '%s'", proc.upload_directory);
+ /* Never returns NULL; it will die if an error occurs */
+ struct abrt_inotify_watch *aiw = abrt_inotify_watch_init(proc.upload_directory,
+ IN_CLOSE_WRITE | IN_MOVED_TO,
+ handle_inotify_cb, &proc);
+
+ VERB1 log("Setting up a signal handler");
+ /* Set up signal pipe */
+ xpipe(g_signal_pipe);
+ close_on_exec_on(g_signal_pipe[0]);
+ close_on_exec_on(g_signal_pipe[1]);
+ ndelay_on(g_signal_pipe[0]);
+ ndelay_on(g_signal_pipe[1]);
+ signal(SIGTERM, handle_signal);
+ signal(SIGINT, handle_signal);
+ signal(SIGCHLD, handle_signal);
+ GIOChannel *channel_signal = abrt_gio_channel_unix_new(g_signal_pipe[0]);
+ guint channel_signal_source_id = g_io_add_watch(channel_signal,
+ G_IO_IN | G_IO_PRI | G_IO_ERR | G_IO_HUP | G_IO_NVAL,
+ handle_sigchld_pipe,
+ &proc);
+
+ VERB2 log("Starting glib main loop");
+
+ g_main_loop_run(proc.main_loop);
+
+ VERB2 log("Glib main loop finished");
+
+ g_source_remove(channel_signal_source_id);
+
+ GError *error = NULL;
+ g_io_channel_shutdown(channel_signal, FALSE, &error);
+ if (error)
+ {
+ VERB1 log("Can't shutdown gio channel: '%s'", error ? error->message : "");
+ g_error_free(error);
+ }
+
+ g_io_channel_unref(channel_signal);
+
+ abrt_inotify_watch_destroy(aiw);
+
+ if (proc.main_loop)
+ g_main_loop_unref(proc.main_loop);
+
+ free_abrt_conf_data();
+
+ return 0;
+}
diff --git a/src/daemon/abrtd.c b/src/daemon/abrtd.c
index 124421a..8196d39 100644
--- a/src/daemon/abrtd.c
+++ b/src/daemon/abrtd.c
@@ -21,10 +21,9 @@
#endif
#include <sys/un.h>
#include <syslog.h>
-#include <sys/inotify.h>
-#include <sys/ioctl.h> /* ioctl(FIONREAD) */

#include "abrt_glib.h"
+#include "abrt-inotify.h"
#include "libabrt.h"


@@ -53,7 +52,6 @@
static volatile sig_atomic_t s_sig_caught;
static int s_signal_pipe[2];
static int s_signal_pipe_write = -1;
-static int s_upload_watch = -1;
static unsigned s_timeout;
static bool s_exiting;

@@ -62,7 +60,6 @@ static guint channel_id_socket = 0;
static int child_count = 0;

/* Helpers */
-
static guint add_watch_or_die(GIOChannel *channel, unsigned condition, GIOFunc func)
{
errno = 0;
@@ -193,124 +190,23 @@ static void sanitize_dump_dir_rights(void)

/* Inotify handler */

-static gboolean handle_inotify_cb(GIOChannel *gio, GIOCondition condition, gpointer ptr_unused)
+static void handle_inotify_cb(struct abrt_inotify_watch *watch, struct inotify_event *event, gpointer ptr_unused)
{
- /* Default size: 128 simultaneous actions (about 1/2 meg) */
-#define INOTIFY_BUF_SIZE ((sizeof(struct inotify_event) + FILENAME_MAX)*128)
- /* Determine how much to read (it usually is much smaller) */
- /* NB: this variable _must_ be int-sized, ioctl expects that! */
- int inotify_bytes = INOTIFY_BUF_SIZE;
- if (ioctl(g_io_channel_unix_get_fd(gio), FIONREAD, &inotify_bytes) != 0
- /*|| inotify_bytes < sizeof(struct inotify_event)
- ^^^^^^^^^^^^^^^^^^^ - WRONG: legitimate 0 was seen when flooded with inotify events
- */
- || inotify_bytes > INOTIFY_BUF_SIZE
- ) {
- inotify_bytes = INOTIFY_BUF_SIZE;
- }
- VERB3 log("FIONREAD:%d", inotify_bytes);
-
- if (inotify_bytes == 0)
- return TRUE; /* "please don't remove this event" */
-
- /* We may race: more inotify events may happen after ioctl(FIONREAD).
- * To be more efficient, allocate a bit more space to eat those events too.
- * This also would help against a bug we once had where
- * g_io_channel_read_chars() was buffering reads
- * and we were going out of sync wrt struct inotify_event's layout.
- */
- inotify_bytes += 2 * (sizeof(struct inotify_event) + FILENAME_MAX);
- char *buf = xmalloc(inotify_bytes);
- errno = 0;
- gsize len;
- GError *gerror = NULL;
- /* Note: we ensured elsewhere that this read is non-blocking, making it ok
- * for buffer len (inotify_bytes) to be larger than actual available byte count.
- */
- GIOStatus err = g_io_channel_read_chars(gio, buf, inotify_bytes, &len, &gerror);
- if (err != G_IO_STATUS_NORMAL)
- {
- perror_msg("Error reading inotify fd: %s", gerror ? gerror->message : "unknown");
- free(buf);
- if (gerror)
- g_error_free(gerror);
- return FALSE; /* "remove this event" (huh??) */
- }
-
- /* Reconstruct each event */
- gsize i = 0;
- for (;;)
- {
- if (i >= len)
- {
- /* This would catch one of our former bugs. Let's be paranoid */
- if (i > len)
- error_msg("warning: ran off struct inotify (this should never happen): %u > %u", (int)i, (int)len);
- break;
- }
-
- struct inotify_event *event = (struct inotify_event *) &buf[i];
+/* We no longer need a name of event */
+#if 0
const char *name = NULL;
if (event->len)
name = event->name;
+#endif
//log("i:%d len:%d event->mask:%x IN_ISDIR:%x IN_CLOSE_WRITE:%x event->len:%d",
// i, len, event->mask, IN_ISDIR, IN_CLOSE_WRITE, event->len);
- i += sizeof(*event) + event->len;
-
- if (event->wd == s_upload_watch)
- {
- /* Was the (presumable newly created) file closed in upload dir,
- * or a file moved to upload dir? */
- if (!(event->mask & IN_ISDIR)
- && event->mask & (IN_CLOSE_WRITE|IN_MOVED_TO)
- && name
- ) {
- const char *ext = strrchr(name, '.');
- if (ext && strcmp(ext + 1, "working") == 0)
- continue;
-
- const char *dir = g_settings_sWatchCrashdumpArchiveDir;
- log("Detected creation of file '%s' in upload directory '%s'", name, dir);
-
- fflush(NULL); /* paranoia */
- pid_t pid = fork();
- if (pid < 0)
- perror_msg("fork");
- if (pid == 0)
- {
- /* child */
- xchdir(dir);
- if (g_settings_delete_uploaded)
- execlp("abrt-handle-upload", "abrt-handle-upload", "-d",
- g_settings_dump_location, dir, name, (char*)NULL);
- else
- execlp("abrt-handle-upload", "abrt-handle-upload",
- g_settings_dump_location, dir, name, (char*)NULL);
- error_msg_and_die("Can't execute '%s'", "abrt-handle-upload");
- }
-
- if (pid > 0)
- increment_child_count();
- }
- continue;
- }

if (event->mask & IN_DELETE_SELF || event->mask & IN_MOVE_SELF)
{
- /* HACK: we expect that we watch deletion only of 'g_settings_dump_location'
- * but this handler is used for 'g_settings_sWatchCrashdumpArchiveDir' too
- */
log("Recreating deleted dump location '%s'", g_settings_dump_location);

- int inotify_fd = g_io_channel_unix_get_fd(gio);
- inotify_rm_watch(inotify_fd, event->wd);
sanitize_dump_dir_rights();
- if (inotify_add_watch(inotify_fd, g_settings_dump_location, IN_DUMP_LOCATION_FLAGS) < 0)
- {
- perror_msg_and_die("inotify_add_watch failed on recreated '%s'", g_settings_dump_location);
- }
-
- continue;
+ abrt_inotify_watch_reset(watch, g_settings_dump_location, IN_DUMP_LOCATION_FLAGS);
}
/* We no longer watch for subdirectory creations */
#if 0
@@ -330,10 +226,6 @@ static gboolean handle_inotify_cb(GIOChannel *gio, GIOCondition condition, gpoin
log("Directory '%s' creation detected", name);
...
#endif
- } /* while */
-
- free(buf);
- return TRUE; /* "please don't remove this event" */
}


@@ -644,11 +536,10 @@ int main(int argc, char** argv)
signal(SIGALRM, handle_signal);

GMainLoop* pMainloop = NULL;
- GIOChannel* channel_inotify = NULL;
- guint channel_id_inotify_event = 0;
GIOChannel* channel_signal = NULL;
guint channel_id_signal_event = 0;
bool pidfile_created = false;
+ struct abrt_inotify_watch *aiw = NULL;

/* Initialization */
VERB1 log("Loading settings");
@@ -700,53 +591,11 @@ int main(int argc, char** argv)
VERB1 log("Creating glib main loop");
pMainloop = g_main_loop_new(NULL, FALSE);

- VERB1 log("Initializing inotify");
- errno = 0;
- int inotify_fd = inotify_init();
- if (inotify_fd == -1)
- perror_msg_and_die("inotify_init failed");
- close_on_exec_on(inotify_fd);
-
/* Watching 'g_settings_dump_location' for delete self
* because hooks expects that the dump location exists if abrtd is running
*/
- if (inotify_add_watch(inotify_fd, g_settings_dump_location, IN_DUMP_LOCATION_FLAGS) < 0)
- {
- perror_msg("inotify_add_watch failed on '%s'", g_settings_dump_location);
- goto init_error;
- }
- /* ...and upload dir */
- if (g_settings_sWatchCrashdumpArchiveDir)
- {
- if (strcmp(g_settings_sWatchCrashdumpArchiveDir, g_settings_dump_location) == 0)
- {
- error_msg("%s and %s can't be the same", "DumpLocation", "WatchCrashdumpArchiveDir");
- goto init_error;
- }
- s_upload_watch = inotify_add_watch(inotify_fd, g_settings_sWatchCrashdumpArchiveDir, IN_CLOSE_WRITE|IN_MOVED_TO);
- if (s_upload_watch < 0)
- {
- perror_msg("inotify_add_watch failed on '%s'", g_settings_sWatchCrashdumpArchiveDir);
- goto init_error;
- }
- }
-
- VERB1 log("Adding inotify watch to glib main loop");
- /* Without nonblocking mode, users observed abrtd blocking
- * on inotify read forever. Must set fd to non-blocking:
- */
- ndelay_on(inotify_fd);
- channel_inotify = abrt_gio_channel_unix_new(inotify_fd);
- /*
- * glib's read buffering must be disabled, or else
- * FIONREAD-reported "available data" sizes and sizes of reads
- * can become inconsistent, and worse, buffering can split
- * struct inotify's (very bad!).
- */
- g_io_channel_set_buffered(channel_inotify, false);
- channel_id_inotify_event = add_watch_or_die(channel_inotify,
- G_IO_IN | G_IO_PRI | G_IO_HUP,
- handle_inotify_cb);
+ aiw = abrt_inotify_watch_init(g_settings_dump_location,
+ IN_DUMP_LOCATION_FLAGS, handle_inotify_cb, /*user data*/NULL);

/* Add an event source which waits for INT/TERM signal */
VERB1 log("Adding signal pipe watch to glib main loop");
@@ -792,10 +641,8 @@ int main(int argc, char** argv)
g_source_remove(channel_id_signal_event);
if (channel_signal)
g_io_channel_unref(channel_signal);
- if (channel_id_inotify_event > 0)
- g_source_remove(channel_id_inotify_event);
- if (channel_inotify)
- g_io_channel_unref(channel_inotify);
+
+ abrt_inotify_watch_destroy(aiw);

if (pMainloop)
g_main_loop_unref(pMainloop);

[Denys Vlasenko]

Post by Jakub Filak
+struct abrt_inotify_watch *
+abrt_inotify_watch_init(const char *path, int inotify_flags, abrt_inotify_watch_handler handler, void *user_data)
+{
+ struct abrt_inotify_watch *aiw = malloc(sizeof(*aiw));

Let's use xmalloc here.

Post by Jakub Filak
+void
+abrt_inotify_watch_destroy(struct abrt_inotify_watch *watch)
+{
+ if (!watch)
+ return;
+
+ inotify_rm_watch(watch->inotify_fd, watch->inotify_wd);
+ g_source_remove(watch->channel_inotify_source_id);
+
+ GError *error = NULL;
+ g_io_channel_shutdown(watch->channel_inotify, FALSE, &error);
+ if (error)
+ {
+ VERB1 log("Can't shutdown inotify gio channel: '%s'", error ? error->message : "");

We know that error is not NULL (we checked it in if()).

Post by Jakub Filak
+#define STRINGIZE_DETAIL(str) #str
+#define STRINGIZE(str) STRINGIZE_DETAIL(str)
+
+#define DEFAULT_COUNT_OF_WORKERS 10
+#define DEFAULT_CACHE_MIB_SIZE 4
+#define APPROX_BYTES_PER_ARCHIVE_NAME FILENAME_MAX

APPROX_BYTES_PER_ARCHIVE_NAME is unused.

In general I think the feature of having a list
of deferred archive names to process is not needed.

The purpose of limiting max number of workers is to
prevent either an attack (someone creates tons of
archives) or a bug in abrt-handle-upload
where it never exits.

In both such scenarios the list
of deferred archive names doesn't help - it overflows too.

The only thing this list achieves is ability to raise
the number of processed archives to a largish number
without using lots of memory: remembering 1000 names
is cheaper than spawning 1000 abrt-handle-upload's.

But, do we actually expect to support such large
numbers of simultaneous archives? I don't think so,
supporting 10-30 simultaneously appearing archives
is probably enough. Mechanism of DEFAULT_COUNT_OF_WORKERS
is enough for such numbers.

I propose dropping DEFAULT_CACHE_MIB_SIZE, the queue thing,
and -c CACHESIZE option.
If -w NUM workers are exceeded, just ignore the new archive.

Post by Jakub Filak
+struct process
+{
+ GMainLoop *main_loop;
+ const char *upload_directory;
+ unsigned children;
+ unsigned max_children;
+ struct queue queue;
+};

You have just one "struct process" object.
I would just use static variables...

Post by Jakub Filak
+static void
+decrement_child_count(struct process *proc)
+{
+ --proc->children;
+
+ char *name = queue_pop(&proc->queue);
+ if (!name)
+ {
+ VERB3 log("Deferred queue is empty. Running workers: %d", proc->children);
+ return;
+ }
+
+ run_abrt_handle_upload(proc, name);
+ free(name);
+}
+
+static void
+handle_signal(int signo)
+{
+ int save_errno = errno;
+ uint8_t sig_caught = signo;
+ if (write(g_signal_pipe[1], &sig_caught, 1))
+ /* we ignore result, if () shuts up stupid compiler */;
+ errno = save_errno;
+}
+
+static gboolean
+handle_sigchld_pipe(GIOChannel *gio, GIOCondition condition, gpointer user_data)
+{
+ struct process *proc = (struct process *)user_data;
+ uint8_t signals[DEFAULT_COUNT_OF_WORKERS];
+ gsize len = 0;
+
+ for (;;)
+ {
+ GError *error = NULL;
+ GIOStatus stat = g_io_channel_read_chars(gio, (void *)signals, sizeof(signals), &len, NULL);
+ if (stat == G_IO_STATUS_ERROR)
+ {
+ error_msg_and_die(_("Can't read from gio channel: '%s'"), error ? error->message : "");
+ }
+ if (stat == G_IO_STATUS_AGAIN)
+ { /* We got all buffered data, but fd is still open. Done for now */
+ return TRUE; /* "glib, please don't remove this event (yet)" */
+ }
+ if (stat == G_IO_STATUS_EOF)
+ break;
+
+ /* G_IO_STATUS_NORMAL */
+ for (unsigned signo = 0; signo < len; ++signo)
+ {
+ /* we did receive a signal */
+ VERB3 log("Got signal %d through signal pipe", signals[signo]);
+ if (signals[signo]!= SIGCHLD)
+ {
+ process_quit(proc);
+ return FALSE; /* remove this event */
+ }
+ else
+ {
+ while (safe_waitpid(-1, NULL, WNOHANG) > 0)
+ {
+ decrement_child_count(proc);
+ }
+ }
+ }
+ }
+
+ return TRUE; /* "please don't remove this event" */
+}

Signal pipe is needed if you can't do your job in signal handler
(because of restrictive environment there).

Here, if the queue thing is dropped, it seems to be possible
to do the necessary operations entirely in signal handler -
with a bit of cleverness with process counting.

What we basically need in signal handler is to wait for
children and decrement children count:

static void
handle_signal(int signo)
{
int save_errno = errno;
while (safe_waitpid(-1, NULL, WNOHANG) > 0)
{
child_count--;
}
errno = save_errno;
}

The problem is, child_count-- is not atomic, it can race with child_count++
elsewhere. (On x86, it _is_ atomic. On RISC, it may not be).

The trick here is to have two counters:

static unsigned children_created;
static unsigned children_died;

, do "children_died++" in signal handler, and NEVER touch children_died
anywhere else.

Then, in other place(s) you'll need to do this:

unsigned children_running = children_created - children_died;
if (children_running < max_children) {
...fork/exec...;
children_created++;
}

Note that it is safe wrt children_{created,died}++ overflowing
from 0xfffffffffffffff to 0. Subtraction will still give correct data.


Since SIGTERM and SIGINT don't actually do any special processing,
we can simply not install a handler for them.

Thus, signal pipe thingy can be avoided.

Post by Jakub Filak
+static void
+handle_inotify_cb(struct abrt_inotify_watch *watch, struct inotify_event *event, void *user_data)
+{
+ /* Was the (presumable newly created) file closed in upload dir,
+ * or a file moved to upload dir? */
+ if (event->name && !(event->mask & IN_ISDIR) && (event->mask & (IN_CLOSE_WRITE | IN_MOVED_TO)))
+ {
+ const char *ext = strrchr(event->name, '.');
+ if (ext && strcmp(ext + 1, "working") == 0)
+ return;
+
+ handle_new_path((struct process *)user_data, xstrdup(event->name));
+ }
+}
+
+static void
+daemonize()
+{
+ /* forking to background */
+ fflush(NULL); /* paranoia */
+ pid_t pid = fork();
+ if (pid < 0)
+ perror_msg_and_die("fork");
+ if (pid > 0)
+ exit(0);
+
+ /* Child (daemon) continues */
+ if (setsid() < 0)
+ perror_msg_and_die("setsid");
+
+ /* Change the current working directory */
+ if ((chdir("/")) < 0)
+ perror_msg_and_die("chdir(/)");

We have xchdir() which does the dying :)

Post by Jakub Filak
+ /* Close out the standard file descriptors */
+ close(STDIN_FILENO);
+ close(STDOUT_FILENO);
+ close(STDERR_FILENO);

Reopening to "/dev/null" is considered to be a better practice.

[Jakub Filak]

Version 2:
* used xmalloc, xchdir
* removed APPROX_BYTES_PER_ARCHIVE_NAME
* the standard descriptors moved to /dev/null
* still using queue as we should decide which use cases we want to
support
* still using struct process because I want to avoid global variables
(well, I know that the structure is overloaded but I'm still happy
with this solution because it is less confusing for me)

git am -s -c *.mbox
-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8--
Related to #657

Signed-off-by: Jakub Filak <jfilak-H+wXaHxf7aLQT0dZR+***@public.gmane.org>
---
doc/Makefile.am | 1 +
doc/abrt-upload-watch.txt | 51 ++++++
po/POTFILES.in | 1 +
src/daemon/Makefile.am | 25 ++-
src/daemon/abrt-inotify.c | 175 +++++++++++++++++++
src/daemon/abrt-inotify.h | 40 +++++
src/daemon/abrt-upload-watch.c | 376 +++++++++++++++++++++++++++++++++++++++++
src/daemon/abrtd.c | 175 ++-----------------
8 files changed, 678 insertions(+), 166 deletions(-)
create mode 100644 doc/abrt-upload-watch.txt
create mode 100644 src/daemon/abrt-inotify.c
create mode 100644 src/daemon/abrt-inotify.h
create mode 100644 src/daemon/abrt-upload-watch.c

diff --git a/doc/Makefile.am b/doc/Makefile.am
index 872d210..e0c50ff 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -33,6 +33,7 @@ MAN1_TXT += abrt-install-ccpp-hook.txt
MAN1_TXT += abrt-action-analyze-vmcore.txt
MAN1_TXT += abrt-action-analyze-ccpp-local.txt
MAN1_TXT += abrt-watch-log.txt
+MAN1_TXT += abrt-upload-watch.txt
MAN1_TXT += system-config-abrt.txt
if BUILD_BODHI
MAN1_TXT += abrt-bodhi.txt
diff --git a/doc/abrt-upload-watch.txt b/doc/abrt-upload-watch.txt
new file mode 100644
index 0000000..1486777
--- /dev/null
+++ b/doc/abrt-upload-watch.txt
@@ -0,0 +1,51 @@
+abrt-upload-watch(1)
+==================
+
+NAME
+----
+abrt-upload-watch - Watch upload directory and unpacks incoming archives into DumpLocation
+
+SYNOPSIS
+--------
+'abrt-upload-watch' [-vs] [-w NUM_WORKERS] [-c CACHE_SIZE_MIB] [UPLOAD_DIRECTORY]
+
+OPTIONS
+-------
+-v, --verbose::
+ Be more verbose. Can be given multiple times.
+
+-s::
+ Log to syslog
+
+-d::
+ Daemonize
+
+-w NUM_WORKERS::
+ Number of concurrent workers. Default is 10
+
+-c CACHE_SIZE_MIB::
+ Maximal cache size in MiB. Default is 4
+
+UPLOAD_DIRECTORY::
+ Watched directory. Default is a value of WatchCrashdumpArchiveDir option from abrt.conf
+
+FILES
+-----
+Uses these three configuration options from file '/etc/abrt/abrt.conf':
+
+WatchCrashdumpArchiveDir::
+ Default upload directory
+
+DumpLocation::
+ Place where uploaded archives are unpacked
+
+DeleteUploaded::
+ Specifies if uploaded archives are deleted after unpacking
+
+SEE ALSO
+--------
+abrt.conf(5)
+
+AUTHORS
+-------
+* ABRT team
diff --git a/po/POTFILES.in b/po/POTFILES.in
index 5e14bc8..8d22e35 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -11,6 +11,7 @@ src/daemon/abrt-server.c
src/dbus/abrt-dbus.c
src/daemon/abrtd.c
src/daemon/abrt-handle-event.c
+src/daemon/abrt-upload-watch.c
src/lib/abrt_conf.c
src/lib/hooklib.c
src/lib/problem_api.c
diff --git a/src/daemon/Makefile.am b/src/daemon/Makefile.am
index 33eec63..9369d26 100644
--- a/src/daemon/Makefile.am
+++ b/src/daemon/Makefile.am
@@ -11,14 +11,17 @@ bin_PROGRAMS = \

sbin_PROGRAMS = \
abrtd \
- abrt-server
+ abrt-server \
+ abrt-upload-watch

libexec_PROGRAMS = abrt-handle-event

# This is a daemon, building with full relro and PIE
# for increased security.
abrtd_SOURCES = \
- abrtd.c
+ abrtd.c \
+ abrt-inotify.c \
+ abrt-inotify.h
abrtd_CPPFLAGS = \
-I$(srcdir)/../include \
-I$(srcdir)/../lib \
@@ -49,6 +52,24 @@ abrt_server_LDADD = \
../lib/libabrt.la \
$(LIBREPORT_LIBS)

+abrt_upload_watch_SOURCES = \
+ abrt-upload-watch.c \
+ abrt-inotify.c \
+ abrt-inotify.h
+abrt_upload_watch_CPPFLAGS = \
+ -I$(srcdir)/../include \
+ -I$(srcdir)/../lib \
+ -DDEFAULT_DUMP_DIR_MODE=$(DEFAULT_DUMP_DIR_MODE) \
+ -DLIBEXEC_DIR=\"$(libexecdir)\" \
+ $(GLIB_CFLAGS) \
+ $(GIO_CFLAGS) \
+ $(LIBREPORT_CFLAGS) \
+ -D_GNU_SOURCE
+abrt_upload_watch_LDADD = \
+ ../lib/libabrt.la \
+ $(LIBREPORT_LIBS)
+
+
abrt_handle_event_SOURCES = \
abrt-handle-event.c
abrt_handle_event_CPPFLAGS = \
diff --git a/src/daemon/abrt-inotify.c b/src/daemon/abrt-inotify.c
new file mode 100644
index 0000000..54ab080
--- /dev/null
+++ b/src/daemon/abrt-inotify.c
@@ -0,0 +1,175 @@
+/*
+ Copyright (C) 2013 RedHat inc.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#include "abrt-inotify.h"
+#include "abrt_glib.h"
+#include "libabrt.h"
+
+#include <stdio.h>
+#include <sys/ioctl.h> /* ioctl(FIONREAD) */
+
+struct abrt_inotify_watch
+{
+ abrt_inotify_watch_handler handler;
+ void *user_data;
+ int inotify_fd;
+ int inotify_wd;
+ GIOChannel *channel_inotify;
+ guint channel_inotify_source_id;
+};
+
+/* Inotify handler */
+
+static gboolean handle_inotify_cb(GIOChannel *gio, GIOCondition condition, gpointer user_data)
+{
+ /* Default size: 128 simultaneous actions (about 1/2 meg) */
+#define INOTIFY_BUF_SIZE ((sizeof(struct inotify_event) + FILENAME_MAX)*128)
+ /* Determine how much to read (it usually is much smaller) */
+ /* NB: this variable _must_ be int-sized, ioctl expects that! */
+ int inotify_bytes = INOTIFY_BUF_SIZE;
+ if (ioctl(g_io_channel_unix_get_fd(gio), FIONREAD, &inotify_bytes) != 0
+ /*|| inotify_bytes < sizeof(struct inotify_event)
+ ^^^^^^^^^^^^^^^^^^^ - WRONG: legitimate 0 was seen when flooded with inotify events
+ */
+ || inotify_bytes > INOTIFY_BUF_SIZE
+ ) {
+ inotify_bytes = INOTIFY_BUF_SIZE;
+ }
+ VERB3 log("FIONREAD:%d", inotify_bytes);
+
+ if (inotify_bytes == 0)
+ return TRUE; /* "please don't remove this event" */
+
+ /* We may race: more inotify events may happen after ioctl(FIONREAD).
+ * To be more efficient, allocate a bit more space to eat those events too.
+ * This also would help against a bug we once had where
+ * g_io_channel_read_chars() was buffering reads
+ * and we were going out of sync wrt struct inotify_event's layout.
+ */
+ inotify_bytes += 2 * (sizeof(struct inotify_event) + FILENAME_MAX);
+ char *buf = xmalloc(inotify_bytes);
+ errno = 0;
+ gsize len;
+ GError *gerror = NULL;
+ /* Note: we ensured elsewhere that this read is non-blocking, making it ok
+ * for buffer len (inotify_bytes) to be larger than actual available byte count.
+ */
+ GIOStatus err = g_io_channel_read_chars(gio, buf, inotify_bytes, &len, &gerror);
+ if (err != G_IO_STATUS_NORMAL)
+ {
+ perror_msg("Error reading inotify fd: %s", gerror ? gerror->message : "unknown");
+ free(buf);
+ if (gerror)
+ g_error_free(gerror);
+ return FALSE; /* "remove this event" (huh??) */
+ }
+
+ struct abrt_inotify_watch *aic = (struct abrt_inotify_watch *)user_data;
+ /* Reconstruct each event */
+ gsize i = 0;
+ for (;;)
+ {
+ if (i >= len)
+ {
+ /* This would catch one of our former bugs. Let's be paranoid */
+ if (i > len)
+ error_msg("warning: ran off struct inotify (this should never happen): %u > %u", (int)i, (int)len);
+ break;
+ }
+ struct inotify_event *event = (struct inotify_event *) &buf[i];
+ i += sizeof(*event) + event->len;
+
+ aic->handler(aic, event, aic->user_data);
+ }
+ free(buf);
+ return TRUE;
+}
+
+struct abrt_inotify_watch *
+abrt_inotify_watch_init(const char *path, int inotify_flags, abrt_inotify_watch_handler handler, void *user_data)
+{
+ struct abrt_inotify_watch *aiw = xmalloc(sizeof(*aiw));
+ aiw->handler = handler;
+ aiw->user_data = user_data;
+
+ VERB1 log("Initializing inotify");
+ errno = 0;
+ aiw->inotify_fd = inotify_init();
+ if (aiw->inotify_fd == -1)
+ perror_msg_and_die("inotify_init failed");
+ close_on_exec_on(aiw->inotify_fd);
+
+ aiw->inotify_wd = inotify_add_watch(aiw->inotify_fd, path, inotify_flags);
+ if (aiw->inotify_wd < 0)
+ perror_msg_and_die("inotify_add_watch failed on '%s'", path);
+
+ VERB1 log("Adding inotify watch to glib main loop");
+ /* Without nonblocking mode, users observed abrtd blocking
+ * on inotify read forever. Must set fd to non-blocking:
+ */
+ ndelay_on(aiw->inotify_fd);
+ aiw->channel_inotify = abrt_gio_channel_unix_new(aiw->inotify_fd);
+
+ /*
+ * glib's read buffering must be disabled, or else
+ * FIONREAD-reported "available data" sizes and sizes of reads
+ * can become inconsistent, and worse, buffering can split
+ * struct inotify's (very bad!).
+ */
+ g_io_channel_set_buffered(aiw->channel_inotify, false);
+
+ errno = 0;
+ aiw->channel_inotify_source_id = g_io_add_watch(aiw->channel_inotify,
+ G_IO_IN | G_IO_PRI | G_IO_HUP,
+ handle_inotify_cb,
+ aiw);
+ if (!aiw->channel_inotify_source_id)
+ perror_msg_and_die("g_io_add_watch failed");
+
+ return aiw;
+}
+
+void
+abrt_inotify_watch_reset(struct abrt_inotify_watch *watch, const char *path, int inotify_flags)
+{
+ inotify_rm_watch(watch->inotify_fd, watch->inotify_wd);
+ watch->inotify_wd = inotify_add_watch(watch->inotify_fd, path, inotify_flags);
+ if (watch->inotify_wd < 0)
+ perror_msg_and_die("inotify_add_watch failed on '%s'", path);
+}
+
+void
+abrt_inotify_watch_destroy(struct abrt_inotify_watch *watch)
+{
+ if (!watch)
+ return;
+
+ inotify_rm_watch(watch->inotify_fd, watch->inotify_wd);
+ g_source_remove(watch->channel_inotify_source_id);
+
+ GError *error = NULL;
+ g_io_channel_shutdown(watch->channel_inotify, FALSE, &error);
+ if (error)
+ {
+ VERB1 log("Can't shutdown inotify gio channel: '%s'", error->message);
+ g_error_free(error);
+ }
+
+ g_io_channel_unref(watch->channel_inotify);
+ free(watch);
+}
diff --git a/src/daemon/abrt-inotify.h b/src/daemon/abrt-inotify.h
new file mode 100644
index 0000000..7674fc0
--- /dev/null
+++ b/src/daemon/abrt-inotify.h
@@ -0,0 +1,40 @@
+/*
+ Copyright (C) 2013 RedHat inc.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+*/
+
+#ifndef _ABRT_INOTIFY_H_
+#define _ABRT_INOTIFY_H_
+
+#include <sys/inotify.h>
+
+struct abrt_inotify_watch;
+
+typedef void (* abrt_inotify_watch_handler)(
+ struct abrt_inotify_watch *watch,
+ struct inotify_event *event,
+ void *user_data);
+
+struct abrt_inotify_watch *
+abrt_inotify_watch_init(const char *path, int inotify_flags, abrt_inotify_watch_handler handler, void *user_data);
+
+void
+abrt_inotify_watch_destroy(struct abrt_inotify_watch *watch);
+
+void
+abrt_inotify_watch_reset(struct abrt_inotify_watch *watch, const char *path, int inotify_flags);
+
+#endif /*_ABRT_INOTIFY_H_*/
diff --git a/src/daemon/abrt-upload-watch.c b/src/daemon/abrt-upload-watch.c
new file mode 100644
index 0000000..88d3033
--- /dev/null
+++ b/src/daemon/abrt-upload-watch.c
@@ -0,0 +1,376 @@
+/*
+ Copyright (C) 2013 ABRT Team
+ Copyright (C) 2013 Red Hat, Inc.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+#include "abrt-inotify.h"
+#include "abrt_glib.h"
+#include "libabrt.h"
+
+#include <syslog.h>
+
+#define STRINGIZE_DETAIL(str) #str
+#define STRINGIZE(str) STRINGIZE_DETAIL(str)
+
+#define DEFAULT_COUNT_OF_WORKERS 10
+#define DEFAULT_CACHE_MIB_SIZE 4
+
+static int g_signal_pipe[2];
+
+struct queue
+{
+ unsigned capacity;
+ GQueue q;
+};
+
+static int
+queue_push(struct queue *queue, char *value)
+{
+ if (g_queue_get_length(&queue->q) >= queue->capacity)
+ return 0;
+
+ g_queue_push_head(&queue->q, value);
+
+ return 1;
+}
+
+static char *
+queue_pop(struct queue *queue)
+{
+ if (g_queue_is_empty(&queue->q))
+ return NULL;
+
+ return (char *)g_queue_pop_tail(&queue->q);
+}
+
+struct process
+{
+ GMainLoop *main_loop;
+ const char *upload_directory;
+ unsigned children;
+ unsigned max_children;
+ struct queue queue;
+};
+
+static void
+process_quit(struct process *proc)
+{
+ g_main_loop_quit(proc->main_loop);
+}
+
+static void
+run_abrt_handle_upload(struct process *proc, const char *name)
+{
+ VERB2 log("Processing file '%s' in directory '%s'", name, proc->upload_directory);
+
+ ++proc->children;
+ VERB3 log("Running workers: %d", proc->children);
+
+ fflush(NULL); /* paranoia */
+ pid_t pid = fork();
+ if (pid < 0)
+ {
+ --proc->children;
+ perror_msg("fork");
+ return;
+ }
+
+ if (pid == 0)
+ {
+ /* child */
+ xchdir(proc->upload_directory);
+ if (g_settings_delete_uploaded)
+ execlp("abrt-handle-upload", "abrt-handle-upload", "-d",
+ g_settings_dump_location, proc->upload_directory, name, (char*)NULL);
+ else
+ execlp("abrt-handle-upload", "abrt-handle-upload",
+ g_settings_dump_location, proc->upload_directory, name, (char*)NULL);
+ error_msg_and_die("Can't execute '%s'", "abrt-handle-upload");
+ }
+}
+
+static void
+handle_new_path(struct process *proc, char *name)
+{
+ log("Detected creation of file '%s' in upload directory '%s'", name, proc->upload_directory);
+
+ if (proc->children < proc->max_children)
+ {
+ run_abrt_handle_upload(proc, name);
+ free(name);
+ return;
+ }
+
+ VERB3 log("Pushing '%s' to deferred queue", name);
+ if (!queue_push(&proc->queue, name))
+ {
+ error_msg(_("No free workers and full buffer. Omitting archive '%s'"), name);
+ free(name);
+ return;
+ }
+}
+
+static void
+decrement_child_count(struct process *proc)
+{
+ --proc->children;
+
+ char *name = queue_pop(&proc->queue);
+ if (!name)
+ {
+ VERB3 log("Deferred queue is empty. Running workers: %d", proc->children);
+ return;
+ }
+
+ run_abrt_handle_upload(proc, name);
+ free(name);
+}
+
+static void
+handle_signal(int signo)
+{
+ int save_errno = errno;
+ uint8_t sig_caught = signo;
+ if (write(g_signal_pipe[1], &sig_caught, 1))
+ /* we ignore result, if () shuts up stupid compiler */;
+ errno = save_errno;
+}
+
+static gboolean
+handle_signal_pipe_cb(GIOChannel *gio, GIOCondition condition, gpointer user_data)
+{
+ struct process *proc = (struct process *)user_data;
+ uint8_t signals[DEFAULT_COUNT_OF_WORKERS];
+ gsize len = 0;
+
+ for (;;)
+ {
+ GError *error = NULL;
+ GIOStatus stat = g_io_channel_read_chars(gio, (void *)signals, sizeof(signals), &len, NULL);
+ if (stat == G_IO_STATUS_ERROR)
+ {
+ error_msg_and_die(_("Can't read from gio channel: '%s'"), error ? error->message : "");
+ }
+ if (stat == G_IO_STATUS_AGAIN)
+ { /* We got all buffered data, but fd is still open. Done for now */
+ return TRUE; /* "glib, please don't remove this event (yet)" */
+ }
+ if (stat == G_IO_STATUS_EOF)
+ break;
+
+ /* G_IO_STATUS_NORMAL */
+ for (unsigned signo = 0; signo < len; ++signo)
+ {
+ /* we did receive a signal */
+ VERB3 log("Got signal %d through signal pipe", signals[signo]);
+ if (signals[signo]!= SIGCHLD)
+ {
+ process_quit(proc);
+ return FALSE; /* remove this event */
+ }
+ else
+ {
+ while (safe_waitpid(-1, NULL, WNOHANG) > 0)
+ {
+ decrement_child_count(proc);
+ }
+ }
+ }
+ }
+
+ return TRUE; /* "please don't remove this event" */
+}
+
+static void
+handle_inotify_cb(struct abrt_inotify_watch *watch, struct inotify_event *event, void *user_data)
+{
+ /* Was the (presumable newly created) file closed in upload dir,
+ * or a file moved to upload dir? */
+ if (event->name && !(event->mask & IN_ISDIR) && (event->mask & (IN_CLOSE_WRITE | IN_MOVED_TO)))
+ {
+ const char *ext = strrchr(event->name, '.');
+ if (ext && strcmp(ext + 1, "working") == 0)
+ return;
+
+ handle_new_path((struct process *)user_data, xstrdup(event->name));
+ }
+}
+
+static void
+daemonize()
+{
+ /* forking to background */
+ fflush(NULL); /* paranoia */
+ pid_t pid = fork();
+ if (pid < 0)
+ perror_msg_and_die("fork");
+ if (pid > 0)
+ exit(0);
+
+ /* Child (daemon) continues */
+ if (setsid() < 0)
+ perror_msg_and_die("setsid");
+
+ /* Change the current working directory */
+ xchdir("/");
+
+ /* Reopen the standard file descriptors to "/dev/null" */
+ xmove_fd(xopen("/dev/null", O_RDWR), STDIN_FILENO);
+ xdup2(STDIN_FILENO, STDOUT_FILENO);
+ xdup2(STDIN_FILENO, STDERR_FILENO);
+}
+
+int
+main(int argc, char **argv)
+{
+ /* I18n */
+ setlocale(LC_ALL, "");
+#if ENABLE_NLS
+ bindtextdomain(PACKAGE, LOCALEDIR);
+ textdomain(PACKAGE);
+#endif
+
+ abrt_init(argv);
+ /* Can't keep these strings/structs static: _() doesn't support that */
+ const char *program_usage_string = _(
+ "& [-vs] [-w NUM] [-c MiB] [UPLOAD_DIRECTORY]\n"
+ "\n"
+ "\nWatches UPLOAD_DIRECTORY and unpacks incoming archives into DumpLocation"
+ "\nspecified in abrt.conf"
+ "\n"
+ "\nIf UPLOAD_DIRECTORY is not provided, uses a value of"
+ "\nWatchCrashdumpArchiveDir option from abrt.conf"
+ );
+ enum {
+ OPT_v = 1 << 0,
+ OPT_s = 1 << 1,
+ OPT_d = 1 << 2,
+ OPT_w = 1 << 3,
+ OPT_c = 1 << 4,
+ };
+
+ int concurrent_workers = DEFAULT_COUNT_OF_WORKERS;
+ int cache_size_mib = DEFAULT_CACHE_MIB_SIZE;
+
+ /* Keep enum above and order of options below in sync! */
+ struct options program_options[] = {
+ OPT__VERBOSE(&g_verbose),
+ OPT_BOOL('s', NULL, NULL , _("Log to syslog")),
+ OPT_BOOL('d', NULL, NULL , _("Daemize")),
+ OPT_INTEGER('w', NULL, &concurrent_workers, _("Number of concurrent workers. Default is "STRINGIZE(DEFAULT_COUNT_OF_WORKERS))),
+ OPT_INTEGER('c', NULL, &cache_size_mib, _("Maximal cache size in MiB. Default is "STRINGIZE(DEFAULT_CACHE_MIB_SIZE))),
+ OPT_END()
+ };
+ unsigned opts = parse_opts(argc, argv, program_options, program_usage_string);
+
+ if (concurrent_workers <= 0)
+ perror_msg_and_die("Invalid number of workers: %d", concurrent_workers);
+
+ if (cache_size_mib <= 0)
+ perror_msg_and_die("Invalid cache size in MiB: %d", cache_size_mib);
+
+ if (cache_size_mib > UINT_MAX / (1024 * 1024 / FILENAME_MAX))
+ perror_msg_and_die("Too big cache size. Maximum is : %u MiB", UINT_MAX / (1024 * 1024 / FILENAME_MAX));
+
+ struct process proc = {0};
+ proc.max_children = concurrent_workers;
+ /* By default it is about 1024 entries */
+ g_queue_init(&proc.queue.q);
+ proc.queue.capacity = cache_size_mib * (1024 * 1024 / FILENAME_MAX);
+ VERB3 log("Max queue size %u", proc.queue.capacity);
+
+ argv += optind;
+ if (argv[0])
+ {
+ proc.upload_directory = argv[0];
+
+ if (argv[1])
+ show_usage_and_die(program_usage_string, program_options);
+ }
+
+ /* Initialization */
+ VERB2 log("Loading settings");
+ if (load_abrt_conf() != 0)
+ return 1;
+
+ if (!proc.upload_directory)
+ proc.upload_directory = g_settings_sWatchCrashdumpArchiveDir;
+
+ if (!proc.upload_directory)
+ error_msg_and_die("Neither UPLOAD_DIRECTORY nor WatchCrashdumpArchiveDir was specified");
+
+ if (opts & OPT_d)
+ daemonize();
+
+ msg_prefix = g_progname;
+ if ((opts & OPT_d) || (opts & OPT_s) || getenv("ABRT_SYSLOG"))
+ {
+ openlog(msg_prefix, 0, LOG_DAEMON);
+ logmode = LOGMODE_SYSLOG;
+ }
+
+ VERB2 log("Creating glib main loop");
+ proc.main_loop = g_main_loop_new(NULL, FALSE);
+
+ VERB1 log("Setting up a file monitor for '%s'", proc.upload_directory);
+ /* Never returns NULL; it will die if an error occurs */
+ struct abrt_inotify_watch *aiw = abrt_inotify_watch_init(proc.upload_directory,
+ IN_CLOSE_WRITE | IN_MOVED_TO,
+ handle_inotify_cb, &proc);
+
+ VERB1 log("Setting up a signal handler");
+ /* Set up signal pipe */
+ xpipe(g_signal_pipe);
+ close_on_exec_on(g_signal_pipe[0]);
+ close_on_exec_on(g_signal_pipe[1]);
+ ndelay_on(g_signal_pipe[0]);
+ ndelay_on(g_signal_pipe[1]);
+ signal(SIGTERM, handle_signal);
+ signal(SIGINT, handle_signal);
+ signal(SIGCHLD, handle_signal);
+ GIOChannel *channel_signal = abrt_gio_channel_unix_new(g_signal_pipe[0]);
+ guint channel_signal_source_id = g_io_add_watch(channel_signal,
+ G_IO_IN | G_IO_PRI | G_IO_ERR | G_IO_HUP | G_IO_NVAL,
+ handle_signal_pipe_cb,
+ &proc);
+
+ VERB2 log("Starting glib main loop");
+
+ g_main_loop_run(proc.main_loop);
+
+ VERB2 log("Glib main loop finished");
+
+ g_source_remove(channel_signal_source_id);
+
+ GError *error = NULL;
+ g_io_channel_shutdown(channel_signal, FALSE, &error);
+ if (error)
+ {
+ VERB1 log("Can't shutdown gio channel: '%s'", error ? error->message : "");
+ g_error_free(error);
+ }
+
+ g_io_channel_unref(channel_signal);
+
+ abrt_inotify_watch_destroy(aiw);
+
+ if (proc.main_loop)
+ g_main_loop_unref(proc.main_loop);
+
+ free_abrt_conf_data();
+
+ return 0;
+}
diff --git a/src/daemon/abrtd.c b/src/daemon/abrtd.c
index 124421a..8196d39 100644
--- a/src/daemon/abrtd.c
+++ b/src/daemon/abrtd.c
@@ -21,10 +21,9 @@
#endif
#include <sys/un.h>
#include <syslog.h>
-#include <sys/inotify.h>
-#include <sys/ioctl.h> /* ioctl(FIONREAD) */

#include "abrt_glib.h"
+#include "abrt-inotify.h"
#include "libabrt.h"


@@ -53,7 +52,6 @@
static volatile sig_atomic_t s_sig_caught;
static int s_signal_pipe[2];
static int s_signal_pipe_write = -1;
-static int s_upload_watch = -1;
static unsigned s_timeout;
static bool s_exiting;

@@ -62,7 +60,6 @@ static guint channel_id_socket = 0;
static int child_count = 0;

/* Helpers */
-
static guint add_watch_or_die(GIOChannel *channel, unsigned condition, GIOFunc func)
{
errno = 0;
@@ -193,124 +190,23 @@ static void sanitize_dump_dir_rights(void)

/* Inotify handler */

-static gboolean handle_inotify_cb(GIOChannel *gio, GIOCondition condition, gpointer ptr_unused)
+static void handle_inotify_cb(struct abrt_inotify_watch *watch, struct inotify_event *event, gpointer ptr_unused)
{
- /* Default size: 128 simultaneous actions (about 1/2 meg) */
-#define INOTIFY_BUF_SIZE ((sizeof(struct inotify_event) + FILENAME_MAX)*128)
- /* Determine how much to read (it usually is much smaller) */
- /* NB: this variable _must_ be int-sized, ioctl expects that! */
- int inotify_bytes = INOTIFY_BUF_SIZE;
- if (ioctl(g_io_channel_unix_get_fd(gio), FIONREAD, &inotify_bytes) != 0
- /*|| inotify_bytes < sizeof(struct inotify_event)
- ^^^^^^^^^^^^^^^^^^^ - WRONG: legitimate 0 was seen when flooded with inotify events
- */
- || inotify_bytes > INOTIFY_BUF_SIZE
- ) {
- inotify_bytes = INOTIFY_BUF_SIZE;
- }
- VERB3 log("FIONREAD:%d", inotify_bytes);
-
- if (inotify_bytes == 0)
- return TRUE; /* "please don't remove this event" */
-
- /* We may race: more inotify events may happen after ioctl(FIONREAD).
- * To be more efficient, allocate a bit more space to eat those events too.
- * This also would help against a bug we once had where
- * g_io_channel_read_chars() was buffering reads
- * and we were going out of sync wrt struct inotify_event's layout.
- */
- inotify_bytes += 2 * (sizeof(struct inotify_event) + FILENAME_MAX);
- char *buf = xmalloc(inotify_bytes);
- errno = 0;
- gsize len;
- GError *gerror = NULL;
- /* Note: we ensured elsewhere that this read is non-blocking, making it ok
- * for buffer len (inotify_bytes) to be larger than actual available byte count.
- */
- GIOStatus err = g_io_channel_read_chars(gio, buf, inotify_bytes, &len, &gerror);
- if (err != G_IO_STATUS_NORMAL)
- {
- perror_msg("Error reading inotify fd: %s", gerror ? gerror->message : "unknown");
- free(buf);
- if (gerror)
- g_error_free(gerror);
- return FALSE; /* "remove this event" (huh??) */
- }
-
- /* Reconstruct each event */
- gsize i = 0;
- for (;;)
- {
- if (i >= len)
- {
- /* This would catch one of our former bugs. Let's be paranoid */
- if (i > len)
- error_msg("warning: ran off struct inotify (this should never happen): %u > %u", (int)i, (int)len);
- break;
- }
-
- struct inotify_event *event = (struct inotify_event *) &buf[i];
+/* We no longer need a name of event */
+#if 0
const char *name = NULL;
if (event->len)
name = event->name;
+#endif
//log("i:%d len:%d event->mask:%x IN_ISDIR:%x IN_CLOSE_WRITE:%x event->len:%d",
// i, len, event->mask, IN_ISDIR, IN_CLOSE_WRITE, event->len);
- i += sizeof(*event) + event->len;
-
- if (event->wd == s_upload_watch)
- {
- /* Was the (presumable newly created) file closed in upload dir,
- * or a file moved to upload dir? */
- if (!(event->mask & IN_ISDIR)
- && event->mask & (IN_CLOSE_WRITE|IN_MOVED_TO)
- && name
- ) {
- const char *ext = strrchr(name, '.');
- if (ext && strcmp(ext + 1, "working") == 0)
- continue;
-
- const char *dir = g_settings_sWatchCrashdumpArchiveDir;
- log("Detected creation of file '%s' in upload directory '%s'", name, dir);
-
- fflush(NULL); /* paranoia */
- pid_t pid = fork();
- if (pid < 0)
- perror_msg("fork");
- if (pid == 0)
- {
- /* child */
- xchdir(dir);
- if (g_settings_delete_uploaded)
- execlp("abrt-handle-upload", "abrt-handle-upload", "-d",
- g_settings_dump_location, dir, name, (char*)NULL);
- else
- execlp("abrt-handle-upload", "abrt-handle-upload",
- g_settings_dump_location, dir, name, (char*)NULL);
- error_msg_and_die("Can't execute '%s'", "abrt-handle-upload");
- }
-
- if (pid > 0)
- increment_child_count();
- }
- continue;
- }

if (event->mask & IN_DELETE_SELF || event->mask & IN_MOVE_SELF)
{
- /* HACK: we expect that we watch deletion only of 'g_settings_dump_location'
- * but this handler is used for 'g_settings_sWatchCrashdumpArchiveDir' too
- */
log("Recreating deleted dump location '%s'", g_settings_dump_location);

- int inotify_fd = g_io_channel_unix_get_fd(gio);
- inotify_rm_watch(inotify_fd, event->wd);
sanitize_dump_dir_rights();
- if (inotify_add_watch(inotify_fd, g_settings_dump_location, IN_DUMP_LOCATION_FLAGS) < 0)
- {
- perror_msg_and_die("inotify_add_watch failed on recreated '%s'", g_settings_dump_location);
- }
-
- continue;
+ abrt_inotify_watch_reset(watch, g_settings_dump_location, IN_DUMP_LOCATION_FLAGS);
}
/* We no longer watch for subdirectory creations */
#if 0
@@ -330,10 +226,6 @@ static gboolean handle_inotify_cb(GIOChannel *gio, GIOCondition condition, gpoin
log("Directory '%s' creation detected", name);
...
#endif
- } /* while */
-
- free(buf);
- return TRUE; /* "please don't remove this event" */
}


@@ -644,11 +536,10 @@ int main(int argc, char** argv)
signal(SIGALRM, handle_signal);

GMainLoop* pMainloop = NULL;
- GIOChannel* channel_inotify = NULL;
- guint channel_id_inotify_event = 0;
GIOChannel* channel_signal = NULL;
guint channel_id_signal_event = 0;
bool pidfile_created = false;
+ struct abrt_inotify_watch *aiw = NULL;

/* Initialization */
VERB1 log("Loading settings");
@@ -700,53 +591,11 @@ int main(int argc, char** argv)
VERB1 log("Creating glib main loop");
pMainloop = g_main_loop_new(NULL, FALSE);

- VERB1 log("Initializing inotify");
- errno = 0;
- int inotify_fd = inotify_init();
- if (inotify_fd == -1)
- perror_msg_and_die("inotify_init failed");
- close_on_exec_on(inotify_fd);
-
/* Watching 'g_settings_dump_location' for delete self
* because hooks expects that the dump location exists if abrtd is running
*/
- if (inotify_add_watch(inotify_fd, g_settings_dump_location, IN_DUMP_LOCATION_FLAGS) < 0)
- {
- perror_msg("inotify_add_watch failed on '%s'", g_settings_dump_location);
- goto init_error;
- }
- /* ...and upload dir */
- if (g_settings_sWatchCrashdumpArchiveDir)
- {
- if (strcmp(g_settings_sWatchCrashdumpArchiveDir, g_settings_dump_location) == 0)
- {
- error_msg("%s and %s can't be the same", "DumpLocation", "WatchCrashdumpArchiveDir");
- goto init_error;
- }
- s_upload_watch = inotify_add_watch(inotify_fd, g_settings_sWatchCrashdumpArchiveDir, IN_CLOSE_WRITE|IN_MOVED_TO);
- if (s_upload_watch < 0)
- {
- perror_msg("inotify_add_watch failed on '%s'", g_settings_sWatchCrashdumpArchiveDir);
- goto init_error;
- }
- }
-
- VERB1 log("Adding inotify watch to glib main loop");
- /* Without nonblocking mode, users observed abrtd blocking
- * on inotify read forever. Must set fd to non-blocking:
- */
- ndelay_on(inotify_fd);
- channel_inotify = abrt_gio_channel_unix_new(inotify_fd);
- /*
- * glib's read buffering must be disabled, or else
- * FIONREAD-reported "available data" sizes and sizes of reads
- * can become inconsistent, and worse, buffering can split
- * struct inotify's (very bad!).
- */
- g_io_channel_set_buffered(channel_inotify, false);
- channel_id_inotify_event = add_watch_or_die(channel_inotify,
- G_IO_IN | G_IO_PRI | G_IO_HUP,
- handle_inotify_cb);
+ aiw = abrt_inotify_watch_init(g_settings_dump_location,
+ IN_DUMP_LOCATION_FLAGS, handle_inotify_cb, /*user data*/NULL);

/* Add an event source which waits for INT/TERM signal */
VERB1 log("Adding signal pipe watch to glib main loop");
@@ -792,10 +641,8 @@ int main(int argc, char** argv)
g_source_remove(channel_id_signal_event);
if (channel_signal)
g_io_channel_unref(channel_signal);
- if (channel_id_inotify_event > 0)
- g_source_remove(channel_id_inotify_event);
- if (channel_inotify)
- g_io_channel_unref(channel_inotify);
+
+ abrt_inotify_watch_destroy(aiw);

if (pMainloop)
g_main_loop_unref(pMainloop);

[Denys Vlasenko]

Post by Jakub Filak
* used xmalloc, xchdir
* removed APPROX_BYTES_PER_ARCHIVE_NAME
* the standard descriptors moved to /dev/null
* still using queue as we should decide which use cases we want to
support
* still using struct process because I want to avoid global variables
(well, I know that the structure is overloaded but I'm still happy
with this solution because it is less confusing for me)
git am -s -c *.mbox
-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8--
Related to #657

Pushed with minor changes (changed some error_msg's to perror_msg's
and vice versa).
Thanks!

[Jakub Filak]

Thank you for the fixes and the review.

Jakub

----- Original Message -----
From: "Denys Vlasenko" <dvlasenk-H+wXaHxf7aLQT0dZR+***@public.gmane.org>
To: crash-catcher-***@public.gmane.org
Cc: "Jakub Filak" <jfilak-H+wXaHxf7aLQT0dZR+***@public.gmane.org>
Sent: Thursday, August 22, 2013 12:04:57 PM
Subject: Re: [ABRT PATCH 2/5 v2] introduce abrt-upload-watch

Post by Jakub Filak
* used xmalloc, xchdir
* removed APPROX_BYTES_PER_ARCHIVE_NAME
* the standard descriptors moved to /dev/null
* still using queue as we should decide which use cases we want to
support
* still using struct process because I want to avoid global variables
(well, I know that the structure is overloaded but I'm still happy
with this solution because it is less confusing for me)
git am -s -c *.mbox
-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8-->8--
Related to #657

Pushed with minor changes (changed some error_msg's to perror_msg's
and vice versa).
Thanks!

[Jakub Filak]

Related to #657

Signed-off-by: Jakub Filak <jfilak-H+wXaHxf7aLQT0dZR+***@public.gmane.org>
---
tests/runtests/aux/test_order | 1 +
tests/runtests/aux/test_order.rhel7 | 1 +
tests/runtests/upload-handling/runtest.sh | 1 +
tests/runtests/upload-watcher-stress-test/PURPOSE | 3 +
.../upload-watcher-stress-test/abrt-handle-upload | 2 +
.../problem_dir/abrt_version | 1 +
.../problem_dir/analyzer | 1 +
.../problem_dir/architecture | 1 +
.../upload-watcher-stress-test/problem_dir/cmdline | 1 +
.../problem_dir/component | 1 +
.../problem_dir/core_backtrace | 4 +
.../problem_dir/coredump | Bin 0 -> 376832 bytes
.../upload-watcher-stress-test/problem_dir/count | 1 +
.../problem_dir/dso_list | 4 +
.../upload-watcher-stress-test/problem_dir/environ | 3 +
.../problem_dir/executable | 1 +
.../problem_dir/hostname | 1 +
.../upload-watcher-stress-test/problem_dir/kernel | 1 +
.../upload-watcher-stress-test/problem_dir/limits | 17 +++
.../upload-watcher-stress-test/problem_dir/maps | 19 ++++
.../problem_dir/open_fds | 9 ++
.../problem_dir/os_release | 1 +
.../upload-watcher-stress-test/problem_dir/package | 1 +
.../upload-watcher-stress-test/problem_dir/pid | 1 +
.../upload-watcher-stress-test/problem_dir/pwd | 1 +
.../upload-watcher-stress-test/problem_dir/reason | 1 +
.../upload-watcher-stress-test/problem_dir/time | 1 +
.../upload-watcher-stress-test/problem_dir/uid | 1 +
.../problem_dir/username | 1 +
.../upload-watcher-stress-test/problem_dir/uuid | 1 +
.../problem_dir/var_log_messages | 0
.../runtests/upload-watcher-stress-test/runtest.sh | 117 +++++++++++++++++++++
32 files changed, 199 insertions(+)
create mode 100644 tests/runtests/upload-watcher-stress-test/PURPOSE
create mode 100755 tests/runtests/upload-watcher-stress-test/abrt-handle-upload
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/abrt_version
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/analyzer
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/architecture
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/cmdline
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/component
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/core_backtrace
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/coredump
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/count
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/dso_list
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/environ
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/executable
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/hostname
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/kernel
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/limits
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/maps
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/open_fds
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/os_release
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/package
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/pid
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/pwd
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/reason
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/time
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/uid
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/username
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/uuid
create mode 100644 tests/runtests/upload-watcher-stress-test/problem_dir/var_log_messages
create mode 100755 tests/runtests/upload-watcher-stress-test/runtest.sh

diff --git a/tests/runtests/aux/test_order b/tests/runtests/aux/test_order
index 87564a1..44d8fa6 100644
--- a/tests/runtests/aux/test_order
+++ b/tests/runtests/aux/test_order
@@ -50,6 +50,7 @@ upload-scp
upload-ftp
upload-filename
upload-handling
+upload-watcher-stress-test
ureport

blacklisted-package
diff --git a/tests/runtests/aux/test_order.rhel7 b/tests/runtests/aux/test_order.rhel7
index 283ea55..935b8fd 100644
--- a/tests/runtests/aux/test_order.rhel7
+++ b/tests/runtests/aux/test_order.rhel7
@@ -49,6 +49,7 @@ upload-scp
upload-ftp
upload-filename
upload-handling
+upload-watcher-stress-test
ureport

blacklisted-package
diff --git a/tests/runtests/upload-handling/runtest.sh b/tests/runtests/upload-handling/runtest.sh
index 70b62e1..4eb9fdc 100755
--- a/tests/runtests/upload-handling/runtest.sh
+++ b/tests/runtests/upload-handling/runtest.sh
@@ -43,6 +43,7 @@ rlJournalStart
rlRun "setsebool -P abrt_anon_write 1"
rlRun "service abrtd stop" 0 "Killing abrtd"
rlRun "service abrtd start" 0 "Starting abrtd"
+ rlRun "service abrt-upload-watch start" 0 "Starting abrt-upload-watch"
rlRun "service abrt-ccpp restart" 0 "Start abrt-ccpp"
rlPhaseEnd

diff --git a/tests/runtests/upload-watcher-stress-test/PURPOSE b/tests/runtests/upload-watcher-stress-test/PURPOSE
new file mode 100644
index 0000000..e6aeb45
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/PURPOSE
@@ -0,0 +1,3 @@
+PURPOSE of upload-watcher-stress-test
+Description: Test upload watcher performance
+Author: Jakub Filak <jfilak-H+wXaHxf7aLQT0dZR+***@public.gmane.org>
diff --git a/tests/runtests/upload-watcher-stress-test/abrt-handle-upload b/tests/runtests/upload-watcher-stress-test/abrt-handle-upload
new file mode 100755
index 0000000..477d739
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/abrt-handle-upload
@@ -0,0 +1,2 @@
+#!/bin/sh
+rm -v $3
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/abrt_version b/tests/runtests/upload-watcher-stress-test/problem_dir/abrt_version
new file mode 100644
index 0000000..098d34f
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/abrt_version
@@ -0,0 +1 @@
+2.0.7.114.gae9b
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/analyzer b/tests/runtests/upload-watcher-stress-test/problem_dir/analyzer
new file mode 100644
index 0000000..1ab966f
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/analyzer
@@ -0,0 +1 @@
+CCpp
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/architecture b/tests/runtests/upload-watcher-stress-test/problem_dir/architecture
new file mode 100644
index 0000000..8790996
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/architecture
@@ -0,0 +1 @@
+x86_64
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/cmdline b/tests/runtests/upload-watcher-stress-test/problem_dir/cmdline
new file mode 100644
index 0000000..4c9b8c4
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/cmdline
@@ -0,0 +1 @@
+sleep 3m
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/component b/tests/runtests/upload-watcher-stress-test/problem_dir/component
new file mode 100644
index 0000000..a4b710b
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/component
@@ -0,0 +1 @@
+coreutils
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/core_backtrace b/tests/runtests/upload-watcher-stress-test/problem_dir/core_backtrace
new file mode 100644
index 0000000..66736fd
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/core_backtrace
@@ -0,0 +1,4 @@
+374add1ead31ccb449779bc7ee7877de3377e5ad 0xbbcd0 __nanosleep_nocancel libc.so.6 -
+3ddc2c6ae42f850be69a87632e80e8453a94ae82 0x3d48 - [exe] -
+3ddc2c6ae42f850be69a87632e80e8453a94ae82 0x33b5 - [exe] -
+3ddc2c6ae42f850be69a87632e80e8453a94ae82 0x14b6 - [exe] -
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/coredump b/tests/runtests/upload-watcher-stress-test/problem_dir/coredump
new file mode 100644
index 0000000000000000000000000000000000000000..8ca8485cd5412170897aac92b2c700ef1551e418
GIT binary patch
literal 376832
zcmeF431D1R_5bfors*_o!=$8;LZQ<VG}JEBmC#aoNs}^#v>lohqJWufZPI2Ll1UpX
z%9Jjq8-zuzh&u4ETKu5|MX8D~>=jWXqE-bAE?6x13y4|-od3D+eor!aWNg{oe&@l>
z_nvdl-9GQWW!@~kvb1c8+vQTcdDQ#Wprw?>1u1H&LX`{K1C4VgB~E8=8|iSs)P0Q5
zc^@3kp4cBoIc#0e?#d5;D2xn*>p4`{%hVwO#D&AiKpgQru7P?M;(BWw_1wnxu^Mw=
zej)gk-<q3Rll$|(***@F#C&#ugCSF1LKUKp8rkOi=kfN*z4K*uN(DLc=&PloCH3D
zT<lyN^?EiAtvA5PZ*nQ~v$+h^yK!he3l&*Y>e>6f5B>E!>iJQR)mcwvm;m3WhxQjj
zJ=T<Z_BcbRm$-M!3s)jGRRYEKu->rFxL&_&=a6~<RAx=-&u%D&{$eAB)<Yck!#c2D
zx>3)6;`zzcar(iFHKjj$zxUXGmFk}~wBA4H&#sX`Jrz#sEftBKchuL>***@8T6J
zOLfd>{RdlWo_)t%sMLq9P)h6m53~1;de?B3-xvl@|2cXLdh`1y*mI6euxy+E+=<K*
zJ|wunp|8DePkrEzqgeG$GVPy+g5JB=*=uhOY;OMm;@-y%jC2p(_v}xjJRi-ti;`Xm
z%vh~NrF;t+BTlyMP4)GWlnC4Q&uR9j_})%dk1aq0)oYx`Hklj5+T$gYeD(0Xy2u<@
zltDob;***@yeqA^kyHgq=mMH>k{Qh!FFE4yzsMdT*G+Fb~%^T7!HJvb=cVs`$UI5
z0rscz&AONb`-O9ieH`ptQSPX2uCufz^|EA!SK1H`jwqO84GE7?<HH=4Lt@@R<IYTU
z|92b>i(ntEkLLRJ_UbkD?W(C!HMOcH8uihtNO=kYeT-OoO;9RZx_nh7*|H{50R0za
z`8;>5Q+c!T$J(yM2>~JS?@8cF$LoJLUjOsGjD=zMuNxe{e*Hg6S<0^_+4oxd9*yHH
z<XDg3CX<!#o%Fdz-**+#^+#`oZcEF6PP8qhSka9AIzaa~xLL>=_2~;4ZLD;+Zri%P
zzUCde#5PI8c7}BqZUXZLZXW%1xgMT&9k|psNyB!AwHpIt<@xF6bgIDoD%=)X+P1W)
z68rC<qI(tkU9?W;`+_E*FKF2r)|b&AE7!Y#>J1FtRQ-p4d7LZJUpg*%xG1oqdo{DE
zv1b0<nN4-mn;Kg>I;VFQ&YwPi?u_=<8MEzq>sxOfd+zd8_BfcMpDaD*TuQoWUB2ka
zX={HwbL;45Z~R#8jM$;l1vh-<DqR+!q}$N*N_buVn76P_`({znuZ13Qf9s=***@W
z-08o4dg>GVfBf35zqowj3$;***@tnVniih34PYv?xfz*Hl<R6#nKz<kLaQ>Im_|Hn?
z&maK9^?O~K_^Nc{N#pNH<DZ<yUqt;5H=***@u4*1yeEzS$u$18Y5Zkr`qle-IR3v$
z<Nsh9zdoFXYo{lT|AjRE&!!pYh&29hrt$wIP5TtC{FI#ANom^ILxIEX$AUERTPS`w
z4)0Cl*H7<;i?2)LKS=!!H=aXj#&dm|b}G}vk4fX7na01C{KJh$KNcEpUY|@8e`cEY
zOVh;b*H+`ySjEqXG<Usa!5=!ovh6=meZA?&$2sa`6{!x}wmwJd?ay}mC#ypE`MmwQ
zJ)W^DaFS)mebfG{)G2m56IHh(eyZ(1PQ`IOnTdEGsrv(&E0(w1b2(fpMzPv=-nn1>
z5&J}iz`H}ht(^a54DonuYp74x(9h|(X68W7O?M0I)pcA${?qppBkae5dfn?SI-!vs
z-({+$)S&j=aP24Pc4LHE1v|H~ksjB1R2cU0ww<BY!tOXp*LSNZ*?Oz23frphS#;tp
zoZ?S?***@voL=jXe+P9()Ymm4ANO!1$R;_piF(<8irtvkg(Yrw12o)rJ=6n{L$KQG0<
zFU3DU#ov?S52X0_r}zh+Pg8j!#XmhI{y>VK&rNo5IBbi41`EWt?M?CPdqv)SDSkeO
z>ax5Pf9mT8e~Lf#Id4IVpU=6voj{78pZjWmVTwOXJFr<P{?zBf!4!Y$^Wad5KlQnA
zWs0Al_v$XfDSlql+TW1kPrd#{Qv9jczs?l@@wx$***@t|)XXmXe#Xp{%q}?h06H@$p
zQv9jUx8o`Pyp;HTDSm$bt&4k7{CH>4uC+hKe{xDYi4_0D6#sz~|EVed-W0z-#eXox
zKRLzUm*St2;y;w)KP|=***@F?@gGU?zbD17s>8OW*QUNl<IS7mFJLFBFU5aGia#&K
zKQ+bAS<PlH9T?ruB$GDRqHN!0_wuZ4Z;`!h^;j!VUZ^XT=<;$ZUpji(K$LmCeg(zt
zqjoA0x6Y2bkp7mA*8SW><%g);UI%tR7i9A>rpxa}xxF5!pTSX>_Mb$#J&tAD+?z8>
zvi&ut30?m+D);_6nY7oR-F|4GeSI#SNGE1pK2_g)QTaA^QFfkg#B$%2m7TXZ+gFtB
zJ$LkFlwV@!34uL-eM0|DGWk^!wQSpn&6&$~XYBGU&9<&~yVuh67LQhEQ_&R^w};{~
zX&e-nxq0M=vb$Z^xUbIG?b$UVV}Xl8&mZmn0?B!!-K&gs2}#*#`*rc%RKMX+GU=!4
zyy;`_3GzqCU$31d*<p7$+pFjMJoyihf2;PNo89Mfe`H|3x_k^Rp2Fvn$<4al<}nzc
z+nq+``>5RJd49Hi8qxDDA%8FVuhaFGZOhn9Q}yq1yO+>VOK>tPqoO8?3;rRQ)URn&
zakg)_yEHp*S4L5`f19TyyI}JOO?G=$b^&>aG({BaWkoY~c&NJW_ik$MAhow$w|Bv=
z5!*7WvaQV{4M2CLJvTk>-%^})IGJ?oIBeSg8u<&!pL%|<i>s>EG`F@?wMVPlqE%J&
z)=^EZequ3SRaHgW8e5{Pt7;q8srLG4Q)_K?***@v`gX^XaQs8#LJwwBuFh-!^Q8(ZqM
z*;d=ors`_f)JLQBol(_N-O_5_39WByYi*<UsE^upy1A;=)s0Q6v3-NxgKDd<Zlcyz
zbxmtqR8`e?HbzyXt#wUXb#ql?%X;cJ+DJoescx>PR;t_E>uIQV6IE5U(aR(CRn%uy
zb4^ukM_W~Mb*G|%>h@?%P4z9RrG5jCy}p?yLT%89T3V~LSGTO1>znmds;X9}Oj$c4
zm#fvaO|9)TZPR3PO?zVt4Uapn(Tz|?b=0DwQ4kICw5(C9H?%cI^>9~5>Raj>$urE1
z+8b!XW?nS}QDduXZLV%?(NuMdRaJU_VCUEcG^c25T~(7>9qEX+S9P>d|Lf}O)atf+
z-J9+NoH$#xx}&K{54f|ss+zi>(O+2K)~@GmPmNlyYwd`t4*lMt(&aYWj<))NA=@pk
zH<haDqqSAFt&z*y8rL*Ll^&cPkvg|*Y4PH!*)!(Mm^+Z4HJBaD%^4`3J!9TKd}fx1
z^Z)5s$K9<B_Iv(ehrg%U`1v}^@!>Ax=j$o^`}Y_>UpLv`d!6y~b&&n>myJL0g87_`
zK1J#+f%j(%9sVQmTMj?hi#hz=i0^gy_rqWC$D#c`&;2_5ark>2{(bO!|***@AJe
z{(kuT9R5A<`|)`p<`a2nt-W6g9RA+#8*Y}vpLo#32OWOj|6spNGM~4iQ<lAja*cn#
z)1PJh2OR$***@dS@cVO&|DeO)?KA!(@bi9RKZYS91cZPP_^%;=)A_)Mgq(J|iT<+K
z8n&nb-o7;D>xZrNM&}<<|1Z$jpzBI4rMFRwjpjfP7a2PbwvN#opU3$b06$|2_$<4e
zpD{GN-`Ictd$Ls*>~R}=gf?h<ex+qAy`-?WgKo=Kex?z_%{D*F$***@k8;%5Ni+2(qd
z0so<@u$|+3bpE)hzE9U1U-$T#Kzxif#2*@MY<_(h!bJJm3-?c-AG7DvpJn1{|JpWl
zq3gVD$58$X=C8aa{;#i_@&dberH-JyYn3Vg3(C8AB8W>TKCk(_aNSGhK7^m&^YePH
z*M<***@uz|Zmdxjnz15J!9%?R(K5x9e1Xzj`&+1DEr4-@mcezOU&!sm5qs
zyRET=`Rr-p;x)!U+r;^k(0f<ges#pu3q$=+8(X!Q`;ETVh#i->ve~pd*4!6ah)duz
zu5sMQs{b|h1D`VWd&V0(wxYwX?{6^U;rpM^6=u94+$Zt<PvCx2uX_pl|80vI?@ea>
***@lP2|JYnV$3z|3!<LjB)YR3B~%m?5)i!D3H>kDobt`2k7C<oLSKbu2?2CW%$%x5~|
zzcA}zutah~KnMr{As_^***@LO=)z0U;m+-adiJXrjKQYE}7+j_B&?g+m(MG{mCM
z8T8iJo*ro8$HumvFscq3?SsaD0YBRD{***@Umr1ATnLOC=C50tpyME?Eu_p0lj
zGX-wP_}J9X`E>pRa7=rTsW<Hm=uC&sgUxyt?4#%nDDQ#t^RqFxnEJV_UodtM<GB#Z
z&jtKnGUZdC`Ox=33!qb>)1Wh;0qATfw_6DNY^Vh-fd-*v&=9l&S_!=b8iv+D>!7Ql
z4bZjFCTI&Z0&Rl^z=1!bHXAm7CT=cl{!H3@*k?j{wVnn0QGDnrj`k0{Xt?IZ9<bMq
z^@vgCv+wH<4*8ybkHfwb_Ce?qcboX$2cS<F4gSn%;^#&SpEBC>veAMu583_1k26|1
z&S>a#=zODzVxzswj6Q#jQSW0$`FP~xp~w4kyItmTwBv!-XD{kK4t)YT101J8`{0ME
z?sFWE{V4wg9?TxCfBRu9HFjr~8TWqZ-OvQ|LFfVKW6)mc&!A63pM~~AUxBL8rk_#J
zvCtgoc<2P^DbUH#)1hZTr$c8!=Rwbc7D3O0hM*ThKM1XY)<***@nJD?wdZi8L}y$<?u
z=og_q(62%7b!Y<iebCd#7%uOD^8TF$oAqqiWzb8YP0%Z#yP!8g_d}n6z6_l>*7R#Z
zFM-CO-OxC+2bzHPLi?coP?***@k^+Em60Mvqpp<AFgLGOe<2t5dW9_l^Lv>$+mpi$@^
z=w4_K^aymK&(tq~Rzfd<hN1mAChleE5$J-u1Ll-9&Xl{LZfFM7104bNLu1fqu%r3>
z5y*bnK7XwL+c>-AXvZ&qM%06Lx={ZH)^Vo2yyp+t?Ow6=yS7#ydh{P^Wandcd8h9u
zw)S3W^***@Djb?_QvM}BLxtL|rZTvxr(IL<@&;l$4K>&CoKh5u;t<MUtWc*A)W^djg7
zpcg|w1g(NrLu;V5&^l;6bTxDhv;o=(ZGtvKTcEAb2=p>&8*~Hoa_C0rhoM(MKLY(I
z^h#(9x(Rv}bTf1dbSrclwBx;ILEL-KpY694o%SBYF<-vE?***@4p_D^Y#5`
z`;Xtxd<^xTfIbQBgZ>6u?dY!`{weSutv`Oh^F*w#gQzzq*Q}pSj`scVFFVHe`F+w7
z)GLQpKr5k_LTjOGpm#X>kHG)wW9*;bKfVq1ehz&K`V929&_mE5!***@3Hmqa>rmHt
z!)XN6x&M9eFFeNa^8TNOdWBF6x)@ptb?*NV{QSQ3(e^)=hq*l{hO2($<d5<WtQW4&
***@vQv%#RtcJ)cY~?CFs->48Qk5OQB1k=R#LN%b^v}RnQBeAB5IJe+A`nO-3f&>qRLy
zfc{l{***@KhBj|7?|Z|pZEdUXh&DF0!{+_v^m8-MB(***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+
zgn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwWKe;t7<OUstHU9N$7WvKTnZS|etx8{;s
zT(Z;6Ir0^%***@WWzHPMb8F;Jp2rXFkHBvX%-{C2eNBa>3#WR#y=Z1)#{9>}%BRON|A
zxt@;J^$wk4>K!`8)MM?zR9L;J?jhCb0Ll-z%s5%|***@y^%nYFv`**iUNjZ!UYu-?
zb3pZAbUBpkT|o8VG66^V%i~;${?c*jMF*^yjy*SxEQ8Xy%U7xQ|***@B3)vieHy(
zs-3rC?cb*M20o&PkV9!AB|}fC36#&Kq?^_IaU`Yjlz2QkK8uo`%P2}dO2<(eP06{9
zBgaW_>T&4pWFrHA#~^n+RPWxg$e#dZ(eue;1FFZP$F9e#$LRMIsXV`$yWOp<5hG%r
znA)g<S+4x6tOrk1Tf9>!#E#eF;dQF#sK>9zqWjkK*YnbC>pAGL>Av-N^n8A{_HQ?b
***@Dh>=Zl*Da*)!;`af~S!`Xd{Zu!oo{aHW#cK#w?***@8eA=Y9Q)=PthX`X5DJc<F>^
z|9o2A-DfX-y0Y_=>wa~|!mWoVu8W=h)o+b&FaF=hcU-Y#LG!k=zkJr%C%^f#&#hZr
***@VR|A<n|wSE$O1J{^6EEqxO&H<I9A8jAR`7eJ7iBsrQ}UK1vHCN4=Od****@Y5dVN
z{u1&}q4oo-P5V>P)bB{OKfcEJKM((L6hB$7pny3p`FNO0em&0ksV4q%vh}u*G>78D
zvrT+~aVxDk^xpv--+Eu`?JP>V-@ZE2&a1YUj#E<IzTa>BJK_Hv;`{53e<A$eAl2=6
zO*H*-`;#!v$nhq=5eLc(6tCM4pJe<z-!H)5{fgNyd|c`2<>*!9H|=-ogNC*b+4Ftl
ze(9t#-G1L`ru~(6v{{?luL_L+TKJce+8i)1K8Ef^JCTVd{tOK0C*;@j4deWprw2$|
zEhX*m1!vx0nyc=&|***@6+E%3wrEpb)taW(n(8K1Ro7J2T)nQosy$j=yRNFfv%a<?
zT3yprud1q=8f$9V-%`hp=KAL9rp7fbilnKv)^4G$-p*G?>#L$o?HpO(q8qAerzYEU
zdsI}lraoF#U02s;***@eQs2?W+N#Eu)vc;Fwbh!***@rg2uctyym}x?^-kwHl);
zbxu`nt*Xro$d0yqsE>9;Y|EZ}T_cUE&F)V(O#Qb-8(Y>?A*Q*uwWEa^)qSRPMZHwj
z)zg^N>bClN)***@gVsj;P_b9!gt{OR-O&S-C)F<YHmwzPP0)$AFwXUrSO&l$+gMJ}{*
z>4ilVrF!?#Ic)@;^F4HqbK7~XI>$+uyD7qWv^B!6=Thdsmll3xq(0Z_Yr;~j2`z3a
z?>MP(6qo6Bt2ZqfRC;`4<5*pmPf4Fs^zoA#qeqmR5~I&AI!4FpO^Y7W)*sf|eQ2LP
zXXzLn%***@bMYWHS5vBI%a<&Wc+!K<2kO(ao~6Q|JTG9IQ+2}j6dM;^ZB6A;qSWz
z<8k=oI9`Jee<EPU6LR=N%Z<O%;kWXQKkV>_YK*^c($Mo4-v=FX_>asoasBY?>$$$J
z^L8Y~pL*ZY1HZmz^0iyn*`MMcxJ4s>BE`?wA{~Dq#jmfoy!EE|bsxMPO!23VCpqsA
zxl>f~^ylgCag!{{X}dvJd#*1@+v&g7wq8Nc_1g9?w5`ux&n0a8ac%3{f#-T``%!J{
z)82D&+rD4h`XG6(o9tEhXiDY5?XLzmJ+Dui!JQ+6yLaXLC_C%%;O<4Qe#xbR$?pdL
z`j;KKCkD4Bx^g;xAKc#etpil;@EB{ytYGp!t1A#***@BjE)WNzSFKBJnO}_
zr^%K5O-&@Y{gL2tOCAmGo>y?YOU+9BX)m>Y={?l?_Gb_M`DLZbc8>T#{uIT%+XE}x
zp0G#***@y?Ipmaot#***@nGTmH$T?-@O{d(#FIgVi)}DoWmKx?Jv%HylnS~S8k;-?!GJ;
z{OezfcI5grgA0-!&jfdGvFioT=nrl_;*L(HzZuby!Q_Ds>u^EW4f(zHq<^5}eni>B
zUY*m<9VY~n4|n+p*`V)XHDkvKT_IO+bHY974emxWyBB@<J1+IGrugiv#G!}(M7VaJ
zGU<52wR3vj#3`zD*5k9jdw9*Vof*rvCZelKcXlOcP0aestj7;8Ub-{3pN!y6PawFt
zP({ZDckb`hyLxLf>I-***@E*<zZr)$0*HDKqxHA^B%MwvfFeB(q-XGlju(#x_M?3Z}
zP2RtB(*xZUw)Ct&blec!IqwAOIk<EDiCT6q^3o7#aA)Yj?Hn=nxHS9n)_w2ziu&no
z$G9)2>%RZx`*O|QDf{woFVdda^***@ZWF23}EqKe?|vA?HRB1^YFvC2LyH$QLfoW3ZB
zU~a!6-y7T>&G(h<E}`7+iu`keyQBGm;O;B(3xbc7<OgWiC{q2R!gJ{6K5BUL737Vc
zLPy;c>U373Z2L38?***@9RT-FVPA~***@an|%FeRRJ0cp|***@LF?j6i!OPfJh=0e
zzDZLQ9iP<7?hjkR`+n~^^x0RF$>8Qi^Jz5XEI%i>eZ&J46nw<8uT#P8zUU+x|9NzP
zQ;$1nuh6n^(X#l~Ve0YFy%&*V=lBNdYt~;5PYQ1SR)Ib-MRS6+KS(}ICz9ZO&v=5{
zE4<#|&Xsw=t&ev+8La(***@T{k!r&6mg(*aAZ&NZ$6hNiRnTYq&QT1$-{`n;F^SfeRm
zYa+P);SL(byyqz5-Xgq^)7SA5;*tm_ldn-;m2Jz|>{*%}*yVP&Wo3JpB2M2YG*R51
z>SVHB$CYIJTB*Er^fLNeG+N&i?4t5RRL=LLI!ycblE07q=FUZXsT)dnlRvK}nf#Ha
zUYzZ_#$BA9cXdWtwtu&$IJ;oih?4BUw#=gJ!p$Q~vO^h{xw5hgi?Rbn*#*VfehQ^3
z#o6BT>=EjDjG3U+zPe=***@TayQ$<B$wXwJmykdcy%&)mUhrQ$|xW>x++`m
ztGSdas60mHTXi{gvddkRowqHcDBHi;<EBaH6=nNMv%TkzCO2hi3}&9YsNO-Ur!OMB
z+4Cp=L*yqJ*z`E=Cdb3%|Ah9sS7e9W)G8u$`IA(B_L^k!!`fGt?JLQybT=YGxBD8E
zf0D}Eb$KaOUzhEjM%i3?@#=B^J=vlCG{9}tcHZWUrP)0$_Xq4D7Y+1d?|+K#rFO2<
***@x;Tu*xsep^mhezzs;?Q;(`sy<Qa(D>^9^=)SB*h7nN61`3_xfa|>#2r8c*<h8w}M
zm!3U;ir>%e(Kt7|*9?s7Y--|3DnCHwSE9YE-OIA`c4yGe-{ql2ux$h_hRvDo@~rFv
z+BdWyO6{Ftx3A|***@npKMY)mFM>H2p2q3pQJy*w-1M*z$bpgERMT!i9Q={Rh9e3y`a
zKlw}bdOA0|*Ojrz#RJs!uAuTmYm>>;`xouMf&6{sUqpG{wEs5pd)K|u{~-AT!}y;h
ze<k_#TzJ#%yiEScF#a(#=&oV>`Q(oe<3F4H`-kyYkiVDwS97PNy5A=92b+?~_d5Jr
z$R8%ZbDTGkzjGM>o#gKx#{***@k_YLF!E%_7VFLCty8u^cqpN|<`tjC#qic<aL52eKG
z^*N3F1<lE%KKH1WF(cHdk={|6qel5$n^n%3kLHa1NKW<@Imdli7wdNP=GwTjT)***@Z
***@G~?D=1O7>KtmvzK=f6vmx%v^m*MI3Vfs78v?#8_l82>=!^{s<r|Z+;ehgu&DhYZ
zeAyWr4l3Vq85{bPZ$!q1L(1pL*wC+hnHd|7C|^d#hM3Fe%Gl85@~Mms-7cRyW5XVo
zZ)66w?(=1ApzE`}>Aqj4dGyl!UK&`JPtmE-ENFZFrz+Lcl1yUAc0KK%M*jWe*K^=a
z`%B30Z%rmAvy)W&FCl*+`JZ(7qvThSWb!$Oe;4@=lE2yE-%I|$WwihF-8VM9FYhLQ
zLo}J3sZW^p`G1K%|I_h}^FJNmIR9tNpcy*P{O8ljN$*R~Bzg{Xc{2Gi&4b$6?WXh2
zu8fjw|29uacERQm=Vk}opQEz`dFYf)5p)(hceML64p%X?r{`Tk?R9UY^^<ZU*ZwB*
zAGk7^T!wu?E<1h;`PHU3a=J;!lRxZ;zmxpF&B^4v6u&-***@mSplh+JCM-Z?Dql
zZ8x2W^$AN4&+hMKinF#RlloqjH$A>FemZ`(C6gCw|N32SeFfD1eDe2yESdD^>nt|y
zKif~|ukFdCkDa7?ZB&rIkNlt0?bzo{cMYwt($Tf{-zNH7HhP1-;`9ZeWb_I<e-Zs%
zGJ1)fKbF5GcH{;0ck$>|d|uV-;tA?s?IfPKFSweni*(+n>&Py>KXKk)l5M%~HH*TW
z{mp*0DVcRKaWdVn>`o?6)$_2|mwkWXSYL~Af8qY^u(ubuQPTUTiP~$pI+?tP@^qc_
ze~^Y=Jo*y)TQ>SUOyF!Px`E=<tFI?JG5%fdbM<v_vAzz{baBs6G&*AeE$Y1G*}kQG
zH8T763928aesrI_O{C28<nJW^{Vp;V>+7X^C3UV31AASJnL_6`dhwe2x?J<57mq4H
ze!ec_K5LMJeV<jLIh;#(5e>uLWl=N#r#F87#1Qzo6Whb7$CaUO&7W|-zJVEdmVtvU
zv~$GH7s$`2#5y#fmyF)***@8F8I+Q*9=?}-!>N9Rp*Jo|e91f~VtYVirr!^;qkob;
zzPv*7oQR*B#qpespNsKxFn;dE&$$MkYvDN-KeyuNRNReLJ+B8w-@***@0)D+
z=y~g{>dB{Ge1=};@wWfBsaJRQn)iXU<$uwPpHI~dFBzM8hf&_2Ot$Lyhf%Kc|Ku%o
zx7zPx>HhVxtGE4U8~(<k`@}^<***@a=_b<oA!H;d$@M(rCKXc<}y!;x3pRe*g0#o*&
zfqs{HzWL|B8*V*rV>9PKrm>lGcZRX;dm?IM7iP=zqnCR2#_N60Br`***@7*ZZ*CoC2
zJvE=p{8^@b_Q$=(=5fc+***@v5?=|fABO^`x>#%tq9A|o^(>BU^XYsnZ8}(m7IdeVo
zy5V!)xnI=}v(%qqyFZY*g4=vE%Fl5h#(cSMD7V3W|2dRpl;1DvooH--j?r$PQErEy
z|ML56T;4U?@aO$)%^SMjxL;Q0!M3On5CTF#2nYcoAO!x?2&B88N#}RYFI^npPjP(x
z^-m9ZJ;UXE->vX^EO4T^KjnIbZp+T{eH{BMx0%;(d(Ieozvjc|t~idh>***@lh3C)8R
zLW9u#&;!te&`M|nl-p%>j*tE6bWfUg_QCJP>w+Vv44t1hO}%ver1Kv^JBOfo=#S$I
z9X9*ltQNdO?Kt}lrfIh_jo(@St=a|PWvQX-lwXe>YrC<l&HB}^8*?b>*Ms^sTmr8z
z`=Ar?dXkFh^Tz2neqA;gVaCU=Z@!f7Imlpz|LPpS&gah!|5s1;O-#<h`8LGoU3_kf
zpJxo5-+gZTIo+UbI*Fh!3+n544T+2FA9|c1EJd789>%1U>FiO;V~%jkVIQsB5zg0}
***@3zABm52K<#;~A1p~a;0cHP;&-0TXQ^3!O9N`1zW99th$hdkxZT8DRz<*q3==gmv
zn)Nd<4XmdDySHW_`*t#kG|vm2ZR-7`KgY2#lX70Sdi^)vG5Q&_K0fp_IepF^x(CUT
zvhjZA^Duu-IwfRCw(&dWdG&!y!?vHFM{^Lj!}gIghu$A>***@n5&`jt?Q_rlJD?t$)u
z?uQmY3!w)wo<mS>?^v}H?F6A*ughVx|Ly8Qw0q!PYCqlh4!z6$d8@*9*9FjY<2Y9T
z5sWu4O?yYMzb$3iO*V`-%***@89%0)+3Lj0q^tZ`***@Vt>***@Z0+oHepKE2!0qF!?u
z)yIwgc2&CP{jn?)M`743qqSjMeU+B2M|+;juS5Fq-1lhl&hZ2=FS|4R;bT74_s`BB
zx+H9Ona8j(0O$GR|C-JF!H4(x*nY!4>%ZN8KakJI!9Sb#)A)5Kb9=-2`Rt+RV{YH6
ziHRnz7upAP#<AUn{`WxH&&uOt``=ynV;=j`%;%jNfB#XqzEl1Dv)i%oel_>&vA>l_
zpNGM^=Jgyqd-(O}+co&UyW<pLH#ryY7Z****@kK2!A#?<Zq?PyNjtH>`Yq?OibZ
zxcGb=S%~8!O<Zc<N6Rydqs9F*%bjD|=h#oqa`RoO^>y?$Df)kwI)Zq9y}&l}dky9M
zT!L+$BYmxFCVlg&YX01r#)ao^At=X*3W5J-0{H&_z$uWw|KD?Bjr}<T3zJW5oiu<!
zfB!#=?uXx}rrMup(&t5vvs4)Mj0W)HkX654)SK>5AFGkC=vR?iH`w)9C)!@~8C#w8
z!Pe)6q32=Kk3zrsx1VH3veGAmY+KjU_uIa^Ou6r_8}0sB0~gp~s^GImxt<=ru4f%K
z^{m6D9&0~NRIEHdJvV!Pn194=#>rZOZdi4!-t_rO=hKaoIq-8Qw)uAoc7f+***@MPWF
zpj)T!U_iOQxBEK<>BiY#X!>IXCVOuBvpstM>)$Cj^Sq}{{qn5G?p(Uz(~tbIbHmef
zHvI0(_WY?_*U>S0<Ne^bEv5cM4cM>2D9HW{PQX_Z_q$zP8P9e<RJuzg=*h&0m<lIi
zYV3|N%HvaAC9}OVtgg`6)***@9gYVgr*yZsmw>$gh+fMSVPN>9esuObgH+i>=_WKL+
zmB$}azRK96CB+3QGv>=HrYGJpH7cYo*kRe!X;<g?tS&3|bv5}***@qL@)Ki^yF_IiBQ
zNT0{***@BlYH(8o+a;5uK1UHxrH})LmA%Uhq5wMj;CO<+o!^tXf5yZbXgvkCsSoL
zxMN)<n;LHOsj>cPxn5sQZjryF+tpil+I7}d^u=}+qc7FEz1gAgl&leBs=b$0E0@>X
z=l6|Mms<V|f0puuM`kYGG~ZJGBJ~?${6Syr!N4;Q#=>gMCeIF6*otd<-kJ=ry6Ob?
zESkm5`2{sTU-YCJKhe(etD<mt-xQBKG$q&L?{XKf%gTIm-vq11<97LVRgb$cbaP3`
z822kz(RVIAG0Q*CxA)cqv9g?8f39C`-!w*zxWYGa%>Co&cLO%LT%Iw$5y54Bp1TWT
z$~RumAajbh$z{#jq&(|m-l>5p{@!4w%lhu99Q9pw-7go+_g|G?)VFQ<`49S+D-Zqm
zc~{SaIguqBv%KBYug~vS?kv}s?s993=l)nY;#09)*T`OP%<GMNYgBQ*9;4dPpeDy$
zv5()9)8Gxwr8!&Ux2g)OIK!K_B&I6Hs$hN>EiRvj)|p#{b8}<9(mZmyayQWfF3a8R
z`Oqng&v1=a<@6Q%e5?CD-(~(i$16XP3oGyE6Ju5PDCPF#6ike%Sb^***@xu_N0v9i?t
zs=!;IMs7*m_@~e^<+`U!Sr_>dk<&I!Y#JX|`9IFy>!q(Tj+pA*v2u^!w=`=^{#kj&
z-T5wmx8-r?`e<P__`<QP^hdbM{-D2IJ?9eTa>s&e<5SeAO+Kr5OX1j(({}BNx#p^Z
zm?!r{70OZtSw6q2$n^UBr;oB!RwQF7jYN5^u^~^wz0xzLNV(#9zAmek)<s_I?1H}2
z)bIS(k6bRx@>{-h)tIvG<1*b@@7e42yJ9yeuV<y32IBP>SY1=Pw|!vp1vy=6ujf6R
z-s9Qq_x>gJW4CKZpw7DQiTIEG^d<7V{Kmca9q$X3Y?|#}>GF1~QQ^m8-mI{9tlO8-
***@z~;waE{t{R*%OUvu=*X!tNXVa|1u^-J6%WEAN|K&s=***@fmI%N^{#
zI?tL&N383npr<=F?v6;VrJfHiAFI0bf#<Pe<Gfwh_&k?IZY~YiJk(R2JAwvIC9Wd6
zG|=i-nV(***@R!?<>-y-LElZ!?vZ&6Q7B}U!i_FTJUjK{leynjmAr`*28p4{__V$sN_
zrsh?DoEE8nm%F6eJ<eL`zwSuv@{yDM)_w7>`eQl8)(!O`PeJ0lB}EyYxVyX96W^o?
zws;C3ba}$c8nyM{b$xmWK~K!5T%No!x!qwkdrO!7ImmI}v*M$=e?***@PJD8JGIjXzh
zcOF$7^Qd50oo4;m-RDx{K17T0r&B$rc<-b&*rR;;-hJ1*f(5beu-COz#gtoRsnbjR
zbo|q4N6{Aq>DL+5V%Pm{mzt83xbxAB*u+?<^o-S8!tZUMMfd)3n_|j;OYwC+;t|`D
zS5&C>dp|>aq9~}o6xg9g_^ms<@iF0wxqZGO>TkjpSHc^+@!FDLMj%#mS}f*uc~`1c
zew8&|x%9zOpe)ZO>***@hA-C!(QoY{uiV~5zXZV_mGOkmzGV(XCRV#gASI~<2X$@Fj
z&sFEDi#*EPIcnSp<-Iykv9PfC#E0Mir_ug>)XZMhJ$rV}@5kJnt(JPmP!IQyDS5=>
zy6>u(zB0Jgn3>gA?R_FPwc@>=nD-7kToYZJ{k>|***@r~L*joI%yt930c^4-vcY#Zt
zar)VD?^1U-tme|!s<y53ZQYx9=05*yZ{01Y9q$e_bXjWq7Ud}l<ity8{kZ)(*WA1%
z<}0RQ#JW5+xf%5jCaxN_@#fsvS3YsBzWR>Kj7?PW>FT}XM!KUOcSY8&$vI=x&2&U$
z=BxXvOE=%*d%)v5O=T1*_f=osQsxV&qRgP%Gb7)!f^N0lH|0Ea)h!***@7HhGp`
z^?_20rWN;%SLNfzxV)ZxKYak&rAlc8nbvO#W~t1HQ>^{gmg1n>7h4kBRjST=U)B^$
zxu&)GJ+AmBua8ERac$Q$x2t4J#;99HhU0V%@!jUBA<lX8sTq&Twa&`&Ed8KAZ?enh
z4enFcWrQ*Ae&(***@zVRchae*(D__JpPe;o7Q94@|pvZWT~Pl@+gYEEX(WXoUYRUxm>
zr*dMOZ&***@LfBUE@k)***@6t~+^(eq!Q0HHFso&WDR!TRe1teaiLWSUsJmMtS{H
zx_l>OdQPeS)#3-{=Z8Jktbpsj{H{%^n@&M)m6zpHd9Ibk)IqK*=FR*=uJTQcS&6Zh
z$5-Gwp02sEr2C0k%d(GG?p<!xf8DqVYE0qrDi%C+@LJ37DIDdt0u{OabUyNSxyDR)
z`)@wsDq8d2z8eCuu$raj=Ba(Ic|***@e-w)__&-D8h`b2R&rCdtmDV;zmkCMKNJB5<I
zo1IK4pVE6M6;L{Z()%d!Hk~qfb*KVx&L!2r&!V)D()%f$LrJfu#gz0urA1_aA89#h
zh?2gwq-%-2RnXrHDSeRArIe~E)l$-1J*7328Y!)#)J*9zN>NJdDQ%>51*Iz~T}4T6
zTPb~v(hhCVwu{m=l&+(61Eo(;(%YvfeVWouls-f0W=eZ0-Ad_8l>Ud(ZIo`ObO)t7
zDSe&NU6k~;pVGG|eTPzl(gTzpr1UVQ?^606rSDVvA*CNt(%a*do}ly-N<X9Yb4pKA
z(%Vx;)YGKDqVx=<XDR)j(sPs!Q+l3KKcyEb>Fp&W>SgF(NdHRd6-uwc_qtXW9p#if
zl<0fUioW)&vMC);iMR2TKarCDTKP$oCQ|ZKnnEd`lK$=$om<rVC{3p{lhSNTb15yL
z^nOa`P%5TWLW#E^<;!3%CtX2lC8Y{V`***@WH=6{RqxYD%?~>L}^0fwYlQ6Qx#4
z`nOOvQ2H>Xk5GzH+C*tHr7e`YDD9xMo6<Fux+z^x>Eo34P`Z)QO_V-MDNboGr7uvT
z-;1!f{~_(6bO)ucQu;cjZ&3OcCHh{vx|dRd(tVWfr}Pk|M<^Yj^gT-7r}RTgy_6oO
zq_>}t9;Eb3N>5Sx6{S8(diyQu?<hS>=@6yoC>^HsJf%NU(m%8IXG(vebcB-LUL*ax
z&bw*XQW`-ilhQ~^bc(UJEYh)*awz3eI)PFir3sYuPs~lEbSfo3r74v3b{gsFlnP)^
zC7p)6E}ud90HryU=2OzY0C5(jg_O>ww1|>LsfbcBrBX^uC<Q5<***@etXNcC?gRZzN!
zQYEE}DP2OTic&SDI!g7FR#R%C)JCa;(t1jlQ~C&{k5bx1X&a?3N_4JN*HF5S(#I)%
zg3^tYZld&AO8<|N-ab$IMdWXVewp-kN?(O-BIU0T5CTF#2nd1y%LG1cKNn6cG#bmn
zbMRX`Y=1x0|0QFGq5WSrcJJ**RgcjCG!Bj3ZT#^&jP^nMq2aHh{-Z5+e-YS$uc035
zhhbZH8h->DhxUHQ_!***@Eq5iKMe+U|bCcbI>z40}6zun(9_P}`2FSpAp<***@ZqK$;
z;}2pS4;k%)#vd`ZqF)1>OO+B(uKxt=KK6gt#97b^Xc)@%oT_he^y|U+)$2_^P>%B;
z^WiaO9{z8d=lfiqflLhLmZO|+p}J5Wc9i25R$Yzqn4_Hg`CiuW9M@(We;afMbT9Ny
z=wr}dL0^ITjx+VAK<7jEl^Fk0*kNcJ^rO%lptnKyLm!7e1AP&ye5TzT=uBuY`dbWp
zCA1oPIdmKJCg}e_`%(Wc*gt{(FVvl5`W+9Q0nIBle&$>Z|E16fbQkpV&^w{`Lw^Z9
z0zH15X?F^A4s<#65@<biBb3MSG1xajd!YA2d!f%kM;>qbI|VueS^}+r)<8cB-3^UH
zzXr9y<zCpofc_QQfbwqWe&|7{cZnHK0NMb}&NcqY&^;Undl_^;{2zkd0o?`t0`wl}
zLBu}^+k39*=TERR#v8i;{t)ye_~$@7;g3TPK#xF|A+8460lf-(***@gLf_n^-}{{}tb
z1T&tw(5281LR+98g?2;#AM_sR51_w+CZSm;n)dxcqr9%tsUPw0h0cP`hn@!wLmQxN
z(4Elhp!=ZThVpUxBiK(t4?$msy7J6;M?)t<^Pz>%WzZVvM(Fj>TcP(sd!***@1-O
z5dTmDuYy++nlZubr<0*x=ve4D=n2r1pp&5a&@-Sjpv}-Z&_d`r&=TlU=yGT|v=VwL
zv>Lh^x(?a~-2nXvGzQ%Q9ZEs0bOikK&3$2bp3&G`qZTv_^_^+_5!mV4`*-*EU0~*s
zcAQ5mPh0VwE$I8bxnJe`%<v(H{h+bqJ+***@6mc+J$mT%B>a%6-+Sj<tN+nyrak>0
z$#7ftbW^TBSFX#az^314v2ACa0+h3VG3>u1J_MWZ%U3z#*TSaXgR$Fhh0XW*k<Ov>
z=YHO*UWNAcdp^T$QIykghMn(lby9ubME##Ye+_*Vy5}hK>0fr4d0hlm%Z=@W`k|?P
zrshK{O?=M<M&r;vXfHGYO|7f*{ZUgd+-|fV8iGFtI|6mq4{t)fk3g?98u_Tv&=p2~
***@b_&rb^!jq7~)ai*}nBZsDG=`#6F|_P(S?LP-p$***@iHRokqj3ooao<lsoGO?t&lM
zhjw~6{_CbZUH$***@5(FE-HeelC}_RDtfLnf{d>OWv?^$7G~C|7W(|7WH=@)M&0
zXb774sqx2PJF2VhK2sil+GrRWg9cFE1KU}@***@efJ(`Gy&VGR*orm)(?z>AKHg@
zdN|%^%***@iPBi7+Cm4-GLnj$K0lN?ShW^hsaSGZ44MPJ9O}Pb)LH*EPXdLQnzqiQL
z_d&a%y%yRnHtL7Q;P=4}aXt8*?RQ^k>L;Lqn6Z1HflbDaK%MmyU8cNmmr?7JMgu!w
zUuQISjnTw5qufs4qvra>wo_{}&H27>nR(8~<sRB}z8&TL7Ytv%8|CqfhA;mdmtQ=5
z`F$u)v^dLMnY#ZUp*-?oQ{Iblx-!l9UPO6l`_TGsw<&jx#P#gjq2(Tr+rC6jL3y}(
zX!{<I%f6+UgYrbZv%V?6fa~8lw4D2I;QntJTF(9V9v<q~bN**;!O3s#datk38Vq00
z!+qwtWca6eo{7PSdmZ+ZL+x~PjN^IcANaZEqoy9;->t*_S@@vwhaWd;e*lIWtw6&Z
z|J|u(e&uHw-F~{!TPGSli06`fM;SYRgwa`EqaBF<0?N5v|JA1av2LRUumjLSs09r|
zL(ocS82V0+lRvLB3p{2+=Rox{XT8lcVt=MbXU~LP2t5mW{***@x^|Hkl+A2F&<GtZT+
zLZi$l|60QZ)zgX}>m>0*>`$nI#(!Y^7PRmC#*RV#(8Ob?4^>eA_u$7mat`FJtbg4G
zaHxG)7hi$$b8J>#ug-***@Km***o7Fop-nXjLW|{r_X6rl0`DW|C-T0388gTdC
z^JjY(I&EIRynk5J?Z>ye|Llt!H9oVzaCCCv{rhh9--?F6fpzdL=snQ;p$|hJh5i8g
zIP~YxKIkFnOVC%LZoI!U8hRY`c<71HlcAHKr$OHfeIIltbPn`P=tAgXXc=@Rl-sX_
zeJQjC+6Zljei*t1x*OUJy&k#`dI$7dQ0IO<n$7#0;~H>%=!AAb_dxeS_d^dr4?<st
zhHyTPK=(j>*O>K}fISWN+0ZiRCD11570_MKo1nKr?}k1GeG+;I{dli4{XP%B2iMUi
z*b`v~pe1ahUL~{>8i$6t9q0l0{nwlRg3t)G4C6VPK8*dj;O>BZiZ}rKQRrh1oA=)j
z;E$s}=XvC<o<I0`%^5ghJ&uL*PPIQ0?LCX}^L&n`?^OT#J!`$~IcNCu<I%=(tmV$}
znqCM0zSaH1_XmGI3cq)&e}4YQ>oZ+HZ*_h?AWC9>2e6-xKu^Yf`WetOp`}pXKjjXa
z{j3+E{9<V3W^<p(^LeY|*Po65x7fCr`KYZ!=QS|***@0-`}*5gJKtUot)tQW8ESy3_e
zQ{(T0hJR*k|3Ra1s6sp8XK<g4*HirZiuu0VIxyd(WcqJ1?@NT+>+I)Rkxv>s2JQcX
zv3uCwW9-OhjP~7V)CcWDd+O81AKPcN8ybdkTm<$}hMBfFlx^DgpIkre_<P2n{~V)%
z<Bc9|{C&A5?%f{0^<mRbsLQBt8~SD5;2&Ra{Qcms9$RhCm-#26s1NP?jwug(&u9qR
z4ef(6|2XzT&zsyIZ+ZL?Gv1?(KeWTd`CE-fV27VJ^NPWKtK;***@Gwt?dqJ7x?SXaIY
z#^3E&KmJpVpU3Z9|8I8vyxs8!FwV$-*!X$<^cEPNA-***@P&&O|K7WM=7cVgO5{;C7@
z-)A)N1lE0;^^?GQdb{h#IsVu`Y5cwK()jzQ7=CY-Kc9a>?=pY?ZSU;!-NZtymv`v+
z38B4z*6|+;n(_N?H=5`%%GYn-*NpAFeh2O_e&_isfcD<(_4}=!zXG?JadfXR`gX_9
z_Z9!F@$>P|*YCcyW**F+@1OYk-Hq~i!%+Tw{_;7l-|4P@@AUbL#~;FdM!NAQn#{Pv
zIN$R6;***@KZv5}``HRPYwEL%STo2USy#IKo`1AUIr;gu$_WGx{uddWL=ePOI
za|QnVzVrP*=MB7m%%YQiQ<Oh5?~FMXoBQW}c^uC77oBZ!zdR0Sc{;ngsj<4fzI~uL
zQf|K2Q6FunZyR*00Q_s(T00^G(UgI|p|w4l60Z7Insypn8l!d{6!xw#{+4><=B7eu
zhx{8_+tv*<)sJ=})$Q#Y>NuLdzmE7wTWhqnwzY{XBsjjKrm3-ZUH#>#5=MMmq;{|o
z%K71MuWwu5SUU)!7wxn+RM)j`7_4F;e$X>C$Ln8}MC{iy|0(DH?nd+Z5j-xWFIgK~
zzn?pFhC?miWhO2H4dHW0;m;0jZ#~`+&PMf}u=Vc|=*^dF?t>;6Q~B}xQ~d8gc;O#7
zwm!1a=F9)?+J3kB{V0B}c4SnwUC!^L>^s|(XQ7=n7n#gx*m+QXZhGi{&2tcbPNu(S
zueV$y_U};X?1``=FPQ%LxoD)r*gS64X#z(fp8Ml}cP-Xt;`!fQ<9<$c#***@1pHTqni
z{Z@^I{%CjEHt$FNciF<#rhWc**}epj{***@I@lRNn5t%a+g!hPER{?9T#V3buH?
z=***@x8jP{*mR0WFd{S~{;Jm>Nwj{Z4Ldy7?=xX=;vy|136*+2ct_`6ZQ=XA8^H|@m{
zSNA6qzaQ-da31Q$@f=3|-cwAyFphuzcjOWrkM<I$ntsClrrqb!Zun&G*EqbGr*fJ2
zBN%TS{q93QK9n~gJ^?N~A1}&#F~2b4dNGc~U(7i6VH|z1d(qB5^cPxg#!(67&;L}S
zy*N0iOU&n46Zm{<5855z`I70+***@5|qc|=5_1A7frmM=ZEpo`m{IRp8=Hf
zzrSa}t^;QUI}iQFuwVJQ5=Oth5%c`-5P0{3N7qENe=***@v=%0VDZv7w4al&hs|Ghw7
z2mJ2|^1n0K|BCtj(#U6`_WI?2XOPDq!|$8=***@E~`FH^}k)p+Z4_~Er;zib=5o0
z8vm)^-2M3zzP~rR=F|&***@Bh+{&~p0SwCWQxLw_EB*}5&$woh1l`zP0KpZ`kmmla#K
z=VtyWdFD;YonQA{_RI&C?zr}cn|GZ#>ax9$ti1L8XY6)8uwv3Zm;LgAiOcUi>y_nu
zkK7U4Syu4C3lp9?|K5-N&id+iPd@*T&;9-@S1m4DcX9bI9=hjG1LrsAz+m%vC`~cB
z`3yT$hi~`jttrRv|EHj}hrlhc9`)***@Y72f-***@MtRF&#~e1wdvT%A;}{$Z
z{rk0!a3=?*#9?~*a(*~<&VRU=chWc9c=Y`Axb?MI>nW79Pm^3h$)d!YvwkWY9ix9A
zBz1GfJ8iy?F30Bpb$b+ZC4a6lwXM`V$8&wI$DemhjZMvS{ANdg{5i<f*wj47bA9fQ
zKOdPIo0{kNI~@J-=O$BQQ}Z0p^|?R(JY{NZYW^AY<E+***@vrLUm&2v20=l%vhf0-Jb
zlFz~CBe_1;<IiQLgbvAaJlE%X{CUkG!6`;bn*R86oGGD0vfLln=l%vh-#H|D(D(-4
zxAY%puFKZ(MmeGf*S)<C)yMTDN_>3#aeY<J?@ym-{+(NHPvPIk9kBmJ8Gm2Rke)05
zYVHqM8(_0`!e;G)&AJCR>ps}5`(d*lfX#XkHtQkStVdw8dXJd#vgW~NEr88h2%9ws
zo3#=)YXfZ7PS~tnuvz!***@1F%^S!e%`LoAn56R_`mAKQzyw1+ZBQVY3Ed
zvsS`pZGg?%37fSGHtQbPtovZI?uX5K05<DE*sO<OvmSxX>irw$56yFE0c_Sn*sMX=
ztd+1?8(_0`!e;G)&AJCR>ps}5`(d*lfX#XkHtQkStVdw8dSAu-p?MB1fX!M6n>7fV
zwGuXK18mk#*sNW!S@*zZ-3Oa>KWx?muvrhnW<3O(^$2WM?`xPpG|!<0uvrUXvj$<a
zR>Ed&fX&(oo3#rz>mJyw`(U%~hs}BbHtRvytcPH;9)Zp3eI4_M<~g(gHftel)*x)w
zO4zIouvt4{vv$E|-2<C-A8gkBuvrhlW<3a-^$={<Bd}S$f5-***@8ar%~}YXH3*xv
z5;kiCY}QWLtX;5K_rPY|2b*<2Y}Ny?Sr5WyJp`Ne2y9j_UhlK!@#}wRfkO*nvktlV
zYXjHU0lyD7`p~}f&3(*y_&y5jj5!nRe}{+***@gq`!T)o%ZG#S8+xCmUpMIOfkN}#
z-$c`IKjQUd;>YHG#d-qwEeDPELF2!G9~FJSG<F2q4~;)*{Jy794xRSi`?tJsCH=nQ
zEa>};#(!nvV$d+ug8HEfnt=8_ZR#bUacB%0hFZ|vTTFksUl`i|z<B1tk8v^|pW{A`
z{j6a;***@9=+x%aC)#-aTY|Cx-w;PR{y*>EaIfmzCPQUUDT_?vnw(zs&{IKPpG(Ufa
z87HrE=RCU8%r89R?ai<2pEN%wcjx@#Y37&6e0%eA_AB<ABM^Va90zZ%mP8*b0pWYB
zq~2N_^X2OpUq|?S+dpUM^OZ2ZPu-nYW4|Bp)_Rp%Lg1Yta8K5n-&N14{Ng*)+J9av
z0mu76eWM-s1BrX>`))n6(3E2SxWK?f-S!XW({Gl4^*y$***@2~c58VDJZ8E6AvfH~+l
zPiQ}TSGj(xL;Y^XN%lturrtclw!5GG+BofZ+S(_Z5cp3az^|+L^Evlq{r<H{r~jbR
zIsTOL)1LnP7v50rj2Bx72mv7=1cZPP5CTF#2nd0H9f8=p)O7&g>x(4J_pz)***@qIEp
zPvh6!?^5^DtPl_aLO=)z0U;m+{%Z*6`=;UUot!?lweqvve`g4Z$wNQ)b<AV!t2^dU
z2^0cCKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf
z5D)@FKnMr{***@F~Nz&%-y{5tjbCiXc#r|0}^q5iy~{<mB=W>U`UR<HlYJ0|(+;d}KD
zG3XykaQ^-^x&EWKKKwo#|8DQUdNR)***@vk1wzq<eM$&TL#`Bx7}`WFI1KnMr{As_^V
***@LO=)z0U;m+-***@UIeOsl$+EjU)4fywZokvd<)T7B2nYcoAOwVf5D)@FKnMr{
zAs_^***@LO=)z0U;m+j*Wo%d|G1R(9fw&!@***@b9(qztigccXf}glfkHw=J~d-
z-(^4NR<H*HklS&6!1dHOHa;-=_Lb$+s<ytNTpU8+KafDb|1JK`=zri`WynH62nYco
zFi5}}%nr#_ZW}S=***@O8gn$qb0zyCt2mv7=1cZPP5CTF#2>eG72*u3n{B4QJ
z*4&g4$8R_G$Q0k3$vWHHhxSIlnMNd52nYcoAO!wx3CwJ2teHPo|E-<T-a2ExbD$QU
zBU^`Ww68CF&M3Auuh6LTx$=P>=6b~Ka6R?Bso#%wx}iPL1T=s+3mSs<qP!0phH^h4
zht2+U>O;HEazFgt(***@emn8W7!u{7;****@N6t+%7Z><^DLn7k;+iswUo{cAWk9rD?Z6
zjo(@St=e($w^Z0(hdn3O*qWs#4clbGu^zi-W8LmD>***@S*PrOeu9BqLln%H}y9ct2
zMou!S8$ZQL|1HXJb+tW?{(=5Xy~0)a98AwfyWE;*`q%9Zw|U^~*>1+?gL16<b(4Ey
zmLBaJH}B4&$DfYiQESb6W0g1M@!e&PV;^coplowv?B~***@UVov9aQ?nn-vBpL&h?xp
***@0PPgLo5TBoT98Ukh{>ORE8Gp2LZue*!US-)K9Va!8(wWy*8v8={k8f<$KPxxj
zFM~hV;V**!hHF2ee{yahejfbM2}9f0F>hwO`=v?eC+Yjaf9d#&bC<lCX4LMmS^qp=
zuKX$Z*MkH5KMy~1VE^s#w_#rF|2F*X4*w(YGY5|UG5pMv{ZGL^9-P_#d-zXq_+Nzo
zM2G)X`12h85jYP_aQKgdzZ?6J`#lN%YaRa6;lIw|55RxD!@m%IUMJlC68Jyvh`#{-
ztv{@_*C)q2{VkZMbAPd)>m93JJ<@NV2c7HWM);lUVK4m7{rDC5o#Q{&aY&pH5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@LO=)z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#
z2nYcoAOwVf5D)@FKnMr{As_^***@5Jx90w%fMYHTn|69O+M0j+P~~PuaHi)*
z6Yu-9t(Mnl;VPr5=OjBagX8~8)!H^<=L=UVH52oD{ToB~$0+!7psq9=;&~I!Pf|OL
zUo-VNt`hBY`N7tq?K_+Mt$VWBFDPgUR}ZnT+lu<***@j;F|8=-euxkcRG~?m<X{R1j
zcy_Vfp5CuI`6ie+%Gz5d)Rq6jXr2A`=}T5<b-%9vD)q#yY+HGLFKn{SH~fipPclBA
zk>2Bse)g_%yS<M;***@4WM2HI<)*~^b$=k!E^m9K(Vyy1mw#=gWfyg&%*%H8%{*H(
z&Hm;3cW)cYKQeLX{5g)5_q(<+XP&y!$88n|***@H2}b7g7S61U4Wa2RE%_uK9CoLFP;
z2MhHQTfvKLOBGVPmU^F>s`UQQ$E-z3Z<Y$9p3#8Y_*uQiwD+xt@{zCDm`OL-^;jn&
zjupGf9^1e<!j3WhDD<0uyE!kgD&@DMG!#8n-(9BMcb7T8um&!$!&Jd%jdDFbd|l5v
zZ0cEuO+D6r&}QZN>ABhS!~7#|GfvhLbi=A+^)`}{&ZiqEbBO3*+Sm^LE?TGS?E=qT
z;K{nT!46lf+}{OMkB=!***@sIx|PQ`IM$ta&cxE7xe5?Hst{(;}#uUwwlZ|tssS^o;{
z^rPM5VE3RN_s9F;E1OzOdw(_ip%TjNvkok1IQqH3p<~IOL+N5lEtED;x`GmRn^I^$
zK|Z}P?+)zuF~&XMe<-WL9$%oh#xCLV$oQXc(dThz{ck<H&W`6eRtr4z`1Iy%Z#8z>
z)}Nc>^pkV3Ur^7~QtH*8n)19qO|sAbwHWta4CIp6?zP9&OZ9Y}!i2H?*bfSJBL99n
***@QQw#RGcGanuG!~O&=NP+B!?fCV78)rv!k2CS;Ud^3j#>+OEv5#-gE9JFD9j+a+
zewLvyl55)gH0(UTX>YKu$z@(+>RTuXJNR(_e4gyVyxG5_x!JDD`***@1oJ_=yxw^K
z`!5aKCHp3uaq)h%o;***@37wR?ki(-***@k=***@s*1r2Y?0NfW9qC=o{nx!@)_rHO
zMP8+TP-OHasO_XbJpS{qY_{7;<eU2@<`G6a>{sB|Fy6Em$GEvYR?S0??`WHqCV+W3
zk2fCgs3oa2^Hl17)%P#KMW*~tNBg{<;+byyeu;}?6HGjxm%l#8*aP#{fw9$=vpufY
z0}***@Z;wr`YHb2loiZN3GJ<e~vBGKI=TJ`~HhAv|Y|WN8|sQ6U_0<^_ZuMne~(y
zXK|6O_hfzS#1DP`3zbJ}_nxfxUf49J@{***@MF(|=***@D-XAy6X3!7tNo>+=XdTr
zU4PeNeeFRze0}Hh?F{hY{mJc}34c8_0=*9UMd)47N1#7~z5tzo^JFpfgU|@H8~XpC
z--bQ}9kI|H&vT&XL+hcxfo7d;;-*3ip^KrbpsS&mL$^VH2wk|yw9^9J3cVS6D|A2f
z0qED>Z|WtWPePACJ?EJ6_d<)HVdy&OF6dXG_e1-jyniNhrr%s=_&S$-_3*vAXy6_L
z1$^IHhkAVew3KCgJn$Dn`***@96ua)Qyi%_Hi7qL;J-77n1V%lwEv&X4|}qU`r`;
zmpW)qK=xDlhT|mIFMtbPPN-W^K3LO{o4Dipfa|GmY<yt!?JLWtRc-***@D6xFgR|
zPfQ^o1cZPP5CTF#2nYcoAOwVf5D)@yn?QS0eSJjDX;x)L#Z`+}EU#F(qO7cR<)WtQ
zni*{k(***@ejEo*eXrtPvBZS{2x)zKNXt<9>k<lL$YE0(WZRH#BrLsca!E?Qo;qNt>5
zana&n>7tp@=E%%QW2CCKy0)R-4lXY(FJHQ1dDYUAMe_>h&M#EK73CGni<XrxTHVyK
zdiCY1JXl&*wrFNeW6R8%>h=azQMz*3qPEWUQ59TTUQxdEgQbgR1)Q5IUbVEWWYx;D
zMGettWWmgtb*&p(np&&t^st>{E?KoKR9?FBLYiGmeYCElIWj#kYq-D*DoU2FTr{(z
zy=`VwW6k`zGcSuypEF~QT2WRKx~OE)%=&1pE@-Qdw6;an1r<w|FMeb6%*K}5rjELL
zRkpaQh-OybQnjjlMn`n@^g^{VSW#ZJB2=M)EMBoJR8+CFxU6)MT1CTK)YjS>Rb}N=
zP_|-a`J%S=MS%r%jf(=a7S5TwpsB^q%wN#luqY5%5NTXAH?VNdoCWQzwtwD&I?mM8
znWBY(Sqo}Y@~v&Qf0p()+sy{%***@a!nv~+w0A5*X3e5GfrWGDEU2xvGqV;%H`tlk
z3tA~Ncj4UG3)-***@Z1IUop#r=7fhWIt!}gPvvhvV#({iw+uDJAw0h9rw2@{(^V0E6
z8=I>KjA+wfP_%O}zp*hg5Hslg;DE1g(3#***@FQ))oh$SP%m0DSdsFB1*!hQkh=Ps
zfvRnh=7Id$l=*K+$=9dk+f(vwDf#N!`hj****@Dxzt7)2aPia$Iv#IT^m_rSDi=U
zYa{i8#+t^}0|hnB2AvLy)R-~b`86qd%tia7Ys~J@`OcL1&eWKc_M?rf2Rn<j3=X1w
z{gAvFwr;gKB`8vBR*~&*T|Y4D=9H;5r%tVTc1qB?!F+4Vbebb`***@7(@O+V|}aH
zm%8W6q66_Q9fKhong{dM?Sr#y8zk4(yne8m)lIbn?NqNfhpujZbyI47u+2411L1Xp
z+0NR*o;o+w4TM~tGP2r~+157?##eU^wqD&bh_-djpucs^puf7>jCekcvbw|***@rI$
zrTCj0%@}k6<;_%VW1U$kc0Mv^pmuX)u+i2vW)bL^w!!cX)n+`}UEMk8Z^h!%{`QC&
zr_Oh_N6aL(5vilRefFa~oh1jdGxV_!E80o>luN3kwfeNFPnc6@)Y%IbMpG+|kE`f2
***@gAG%DupkmiX*UubENF`kj;gMe2yl0;dWpkmtsAV}S~u85YaJb=+;uap1?1}^
z9kje)RM%`6$Tvh&qBcZm57|L#@rq))qO7dK_3kKF^NUujJfAK_!4-5lo>^BNt)5xm
zvc7$$T2{1_F3Ibw+h(>$T3efDHdi+`&9tvibjb=1bIqfx_kxt`X03hwho`l+***@l(n
zV1#Ykrg09G493_l-JiZKSYY?8bLzsSD=StNm6a~***@S>GPp%Be8xMIcmRrXaoW%&Be
zWH7X<xTt*S^ad{G)irb-FD+SGv0~+-^^MIcw305$6;<UGAE2vpQ)|l_x;t8a?vP9P
z`6Z=QOUtW5D@)6kE<eAtWYH`|E*e3(jYXBwfQrhOmXsdDoyknvyEEGw>YHX>R!tgh
zpV`#fx~{stv95lm3YE`VbV<u47he*XGv{LaV%dDjrI&QDBcQFp!r5e6mo%_Pm-_Ad
z6+K1&kap%$*=hdNNL5}QtSTx$KU7q1W})wL>O1ReXSPP_TiV+jX0B;zZLXhAx#`vI
z>mt?d?e@(XZW|XhQZd~lc0}6ko5Gc)<)PBW6)C%5QM9e2p6>***@E-WKn`W~{pw5VcH
***@KsSVX_1UXy`I2(8aLX61q}#$d(=?np(_2=z58RVFZtfP$tnbu!V>26QfPP(R
zu3c&Fo2xX>t~Bq>Rhn;Cnt$w-mX}u0s-5O9Us3k|tF`{vwvQ_Qb?hWgn#^HpMWhbZ
zuvoMsb?a)dkTfiAvgS<PBz0o8vSqH9I8C(J4))h=I=ZQqG4P#-mOuIj6p#vtQK6}f
z{|cH!9GB3RPD}-)Au$n&R!T_*MnnAp4D)$+_fppj5)$wS*V^yy-RHe~@7<m6E_eP=
zV7MKFrC)A^rStFK9LiZ7=*0%{z(_E#XPtS6elmJLGaLyGi*EFe5yWnLaOkmt?&HVh
zbi}0DF&#^%u_upt)01uwdPt3NY(Mw=2l~(***@ILnjL(?Hp3vaqv=Kd=Odg(&Z6BKi
zJJu=5R3>ip1%`)${)63}ss}^ifsw%e$JFFYWfEvFmfDeuM^n!x*O{nr^?mcSSDqRo
z!apo%3L8)O!y{ra6c{<E#={QpY&2s|P8|q8_i(_7s6&D}xXAN@%JNZ2zjNav-k+&=
zZbtn~nNp9gd9C82klwjDa39%Xxa4~ZzJpXyC5IPmQvPlIfP^VO^r3rgy2}(=Skx*!
z(~@***@s~41HSl#b~Qt$uZrn<(Tq)DZc|7(Q+eNZjEZW5lvZ+;N&{FL3g!$e~Ro{
zA`IinFLH_QVcBM6*G+byxP1nY%twr7W2Qm*HLEMgt`8W0E;EiPy&AnA_>#tFbp^dY
zBL101rKel-KLkqptMNX~KcVrYmlDYlzFH5xb0GcIc0Q!})q0Fb<j8nB85x^Sjy)SW
z95c<Sc#N64n4B1o9%Eir{n$*#n4U^xX19|XFaG-stR>+mBD=igE4}il{zqJ!{`ak)
z{IPi>%Ji<Hk1CMJhZ=m}%$&1R9VPC6>TB-PAhk<nJO?EnVX4pi2f{|~)4zLh_2f4<
zz4EiC8wxK6?-cfbe9*Sp-Zg}J8Y`~?I1Tz+&{I@~g4aP;L9c-ZF<6>l$OC9_H|RG&
zkAt>j0KEzt1ib>v_Sv2b8MfIL+vcWv=N$NL&_O<RKz}=^;lHoti#5H?b<Z>cvFP8>
zw+O0meW>mMZlMwv%-U;CHx~U0E%lMT%fLNSiA!jG#6zNX$sY>HXNhvVH3#kil1=1q
z2XKp(xS?$AOOCJDYfd#5_gvZYMz3yvH!!RCyal`*`%>@T=DLGYt=Qm~ba0IZWjVNE
zh!m(8v?DI9h-<d8y}_64Sx3#jMheH3qMv`xD0WW)KMTAZ^Rm5x=DNK~ANfiBLG^MD
z$@+cGburm)KkMOl5q;Dq63my$ZBz3P;xbLvs4}`)rbD`(-`PE_`K-59m;YWaCn>w6
z<=@j~`n{o4(&a^6uekra?{fJvl+&pEoME>YK*{11@~(3w*STdB{x4EkSl1^e=3ueQ
zXBC9?rpRyfoGnGlqOjx^4Hv~H7mn797Ev);bZzM|2tO>ctIqrbo)duGEZSVTZ^KoQ
zcX+-`a#lTZb28WO%<lG_LDpub!7MFB%GSL2m_s&zcFq&(L5Jrx(0r5Ubt<}kupoYU
zyGv=$d`;N<oxZCg?-JeROQ*Yzw7RyOpumdl#^uvp*#(>8AN`ZCT^1Iw^TPU|$X;+h
zNk%_kTHcIhSmd8Ct$gX-xf}f`Oi=Q=wtRuqNPl1&xpLN)X(R97Br~7{*|%i&ac7r8
zWPa_NA1%H+***@QQ|6a`***@0tQAmD2W&o_W$@cX%#>b}4M;=h~k}<%l3pU_RQ_
zO?)3sU@}p>{bB#ex-sYI2J>w)_I>$w8F};AzZvbIy-iuCq8_mQG-z*EcB&3A?_YUQ
z+7USU4h1}ry<iKRe2==~=xWgVXwXVJWwiZI`A5le#C&P_dzhVw`O?Zsj``hqXUlN(
zy7}k};cU6qY|WPI%m>9-`TR9>hO3qa*$L{GQDNP>5&1{CTv>kkZW#{Jz{<^=3ljP(
zS=D3Y%9$Uc<7C}c^A%YqU`}O!VfeNPYqS)!Vz-6$H;g=j2duy3E}5Saxw5%+d6V>D
zpHuw!7q^Cg|J|=&a0~0ADBO8e-1>tk{Iy<e!)P0J?$2GzY{HPsLs>p{`%>}zA=vJ!
zU3mxgoq|%o<!D3pt1{cy=J3t0T&)^^IfRO*&UMp{`7>IaTT5Z|aSNnn&>s5H=SjW&
z5*gWGUG$XkoXn~hxtnHf;YO2nAN7D5H!j<qmEZLD_IA3rVwv9Ne%Sj7?=JVQoj#v$
z=dMTGTSKvNw}_gGzumXZFtF24n<+E;jNzT6KdF1ik0sK_;)<Fn!+***@g9k#(***@OA
zJ%z});Y}pXnBk30MkZ3xcq}qG4m_8nhG=|h44J9)STqHEblRLspt;PL33chaj0pZt
zj=v+t;<1Ejc+J=>zHx;hi}3wFPV_5HYZ0#-ygs?H#!>2#-*@YI56kNry-y?^EYIuL
zAuaFJ6x#=hVR>H1=zSp9A#***@IH?u#`bTw?v^rJ5_lNPpk64o1=k;z;GkD+1_0`l1
zd0IP}&+F-uW_0M5*haQrv*jLu_Vz5#=MFxP(Eg<|aom{Kiwyb1^1T0FJC`s}(`xwx
zRq|)`Ip!=GjC7_dVAb|***@LH$G#***@xPep5<FeInp>j)CAXe)G7H6y`f>h
z2urET93!rIjO*0$om!s9i4<N({sqnB{ZmNGhctuv|4N?rtYp&~PM>q-ferk(*)gqv
s{nka}G%_T=lNp-w`Bv`1E9%()N?O%~L^tRS-*+lxDa&$I?EhNvpWa`Hz5oCK

literal 0
HcmV?d00001

diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/count b/tests/runtests/upload-watcher-stress-test/problem_dir/count
new file mode 100644
index 0000000..56a6051
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/count
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/dso_list b/tests/runtests/upload-watcher-stress-test/problem_dir/dso_list
new file mode 100644
index 0000000..2700478
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/dso_list
@@ -0,0 +1,4 @@
+/bin/sleep coreutils-8.12-6.fc16.x86_64 (Fedora Project) 1328878083
+/lib64/libc-2.14.90.so glibc-2.14.90-24.fc16.4.x86_64 (Fedora Project) 1325768966
+/usr/lib/locale/locale-archive glibc-common-2.14.90-24.fc16.4.x86_64 (Fedora Project) 1325768993
+/lib64/ld-2.14.90.so glibc-2.14.90-24.fc16.4.x86_64 (Fedora Project) 1325768966
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/environ b/tests/runtests/upload-watcher-stress-test/problem_dir/environ
new file mode 100644
index 0000000..0bf4fa7
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/environ
@@ -0,0 +1,3 @@
+TERM=rxvt
+HISTSIZE=100000000000000000
+_=/bin/sleep
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/executable b/tests/runtests/upload-watcher-stress-test/problem_dir/executable
new file mode 100644
index 0000000..48f35bb
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/executable
@@ -0,0 +1 @@
+/bin/sleep
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/hostname b/tests/runtests/upload-watcher-stress-test/problem_dir/hostname
new file mode 100644
index 0000000..7646f7c
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/hostname
@@ -0,0 +1 @@
+fluffy
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/kernel b/tests/runtests/upload-watcher-stress-test/problem_dir/kernel
new file mode 100644
index 0000000..9fef50e
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/kernel
@@ -0,0 +1 @@
+3.1.4-4.4.fc16.x86_64
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/limits b/tests/runtests/upload-watcher-stress-test/problem_dir/limits
new file mode 100644
index 0000000..a315e07
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/limits
@@ -0,0 +1,17 @@
+Limit Soft Limit Hard Limit Units
+Max cpu time unlimited unlimited seconds
+Max file size unlimited unlimited bytes
+Max data size unlimited unlimited bytes
+Max stack size 8388608 unlimited bytes
+Max core file size 0 unlimited bytes
+Max resident set unlimited unlimited bytes
+Max processes 1024 30640 processes
+Max open files 1024 4096 files
+Max locked memory 65536 65536 bytes
+Max address space unlimited unlimited bytes
+Max file locks unlimited unlimited locks
+Max pending signals 30640 30640 signals
+Max msgqueue size 819200 819200 bytes
+Max nice priority 0 0
+Max realtime priority 0 0
+Max realtime timeout unlimited unlimited us
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/maps b/tests/runtests/upload-watcher-stress-test/problem_dir/maps
new file mode 100644
index 0000000..ddb48f1
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/maps
@@ -0,0 +1,19 @@
+00400000-00406000 r-xp 00000000 fd:02 131100 /bin/sleep
+00605000-00606000 r--p 00005000 fd:02 131100 /bin/sleep
+00606000-00607000 rw-p 00006000 fd:02 131100 /bin/sleep
+0062d000-0064e000 rw-p 00000000 00:00 0 [heap]
[phone]-3440222000 r-xp 00000000 fd:02 917527 /lib64/ld-2.14.90.so
[phone]-3440422000 r--p 00021000 fd:02 917527 /lib64/ld-2.14.90.so
[phone]-3440423000 rw-p 00022000 fd:02 917527 /lib64/ld-2.14.90.so
[phone]-3440424000 rw-p 00000000 00:00 0
[phone]ab000 r-xp 00000000 fd:02 930260 /lib64/libc-2.14.90.so
+34407ab000-34409ab000 ---p 001ab000 fd:02 930260 /lib64/libc-2.14.90.so
+34409ab000-34409af000 r--p 001ab000 fd:02 930260 /lib64/libc-2.14.90.so
+34409af000-34409b1000 rw-p 001af000 fd:02 930260 /lib64/libc-2.14.90.so
+34409b1000-34409b6000 rw-p 00000000 00:00 0
+7fed84bfc000-7fed8b01f000 r--p 00000000 fd:02 2536008 /usr/lib/locale/locale-archive
+7fed8b01f000-7fed8b022000 rw-p 00000000 00:00 0
+7fed8b058000-7fed8b059000 rw-p 00000000 00:00 0
+7fff4b173000-7fff4b194000 rw-p 00000000 00:00 0 [stack]
+7fff4b1ff000-7fff4b200000 r-xp 00000000 00:00 0 [vdso]
+ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/open_fds b/tests/runtests/upload-watcher-stress-test/problem_dir/open_fds
new file mode 100644
index 0000000..e44e5ea
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/open_fds
@@ -0,0 +1,9 @@
+0:/dev/pts/6
+pos: 0
+flags: 0100002
+1:/dev/pts/6
+pos: 0
+flags: 0100002
+2:/dev/pts/6
+pos: 0
+flags: 0100002
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/os_release b/tests/runtests/upload-watcher-stress-test/problem_dir/os_release
new file mode 100644
index 0000000..9b817ae
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/os_release
@@ -0,0 +1 @@
+Fedora release 16 (Verne)
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/package b/tests/runtests/upload-watcher-stress-test/problem_dir/package
new file mode 100644
index 0000000..d0fb037
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/package
@@ -0,0 +1 @@
+coreutils-8.12-6.fc16
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/pid b/tests/runtests/upload-watcher-stress-test/problem_dir/pid
new file mode 100644
index 0000000..dc903ee
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/pid
@@ -0,0 +1 @@
+22031
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/pwd b/tests/runtests/upload-watcher-stress-test/problem_dir/pwd
new file mode 100644
index 0000000..12ac2e6
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/pwd
@@ -0,0 +1 @@
+/etc/abrt
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/reason b/tests/runtests/upload-watcher-stress-test/problem_dir/reason
new file mode 100644
index 0000000..457d2bd
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/reason
@@ -0,0 +1 @@
+Process /bin/sleep was killed by signal 11 (SIGSEGV)
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/time b/tests/runtests/upload-watcher-stress-test/problem_dir/time
new file mode 100644
index 0000000..8718a5a
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/time
@@ -0,0 +1 @@
[phone]
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/uid b/tests/runtests/upload-watcher-stress-test/problem_dir/uid
new file mode 100644
index 0000000..c227083
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/uid
@@ -0,0 +1 @@
+0
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/username b/tests/runtests/upload-watcher-stress-test/problem_dir/username
new file mode 100644
index 0000000..d8649da
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/username
@@ -0,0 +1 @@
+root
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/uuid b/tests/runtests/upload-watcher-stress-test/problem_dir/uuid
new file mode 100644
index 0000000..84b262c
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/problem_dir/uuid
@@ -0,0 +1 @@
+06c18dd68d22705f2c7fc5b46f517739888487aa
\ No newline at end of file
diff --git a/tests/runtests/upload-watcher-stress-test/problem_dir/var_log_messages b/tests/runtests/upload-watcher-stress-test/problem_dir/var_log_messages
new file mode 100644
index 0000000..e69de29
diff --git a/tests/runtests/upload-watcher-stress-test/runtest.sh b/tests/runtests/upload-watcher-stress-test/runtest.sh
new file mode 100755
index 0000000..3ed2be3
--- /dev/null
+++ b/tests/runtests/upload-watcher-stress-test/runtest.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+# vim: dict=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+# runtest.sh of upload-watcher-stress-test
+# Description: Test upload watcher performance
+# Author: Jakub Filak <jfilak-H+wXaHxf7aLQT0dZR+***@public.gmane.org>
+#
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+# Copyright (c) 2013 Red Hat, Inc. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+# PURPOSE. See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+#
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+. /usr/share/beakerlib/beakerlib.sh
+. ../aux/lib.sh
+
+TEST="upload-watcher-stress-test"
+PACKAGE="abrt"
+
+flood()
+{
+ touch /tmp/flood.$1$2
+ for e in `seq $1 $2`; do
+ for i in `seq $((e * 1000)) $((e * 1000 + 999))`; do
+ echo "$i.tar.gz" > $WATCHED_DIR/$i.tar.gz
+ done
+ sleep 1
+ done
+ rm /tmp/flood.$1$2
+}
+
+rlJournalStart
+ rlPhaseStartSetup
+ WATCHED_DIR=$PWD/watched
+ export WATCHED_DIR
+ mkdir -p $WATCHED_DIR
+
+ # Adding $PWD to PATH in order to override abrt-handle-upload
+ # by a local script
+ # Use 60 workers and in the worst case 1GiB for cache
+ PATH="$PWD:$PATH:/usr/sbin" abrt-upload-watch -w 60 -c 1024 -v $WATCHED_DIR > out.log 2>&1 &
+ PID_OF_WATCH=$!
+ rlPhaseEnd
+
+ rlPhaseStartTest "handle upload"
+ echo "Copying samples to the watched directory"
+
+ flood 0 9 &
+ flood 10 19 &
+ flood 20 29 &
+ flood 30 39 &
+ flood 40 49 &
+ flood 50 59 &
+ flood 60 69 &
+ flood 70 79 &
+ flood 80 89 &
+ flood 90 99 &
+
+ while test -f "/tmp/flood.09";do sleep 1; done
+ while test -f "/tmp/flood.1019";do sleep 1; done
+ while test -f "/tmp/flood.2029";do sleep 1; done
+ while test -f "/tmp/flood.3039";do sleep 1; done
+ while test -f "/tmp/flood.4049";do sleep 1; done
+ while test -f "/tmp/flood.5059";do sleep 1; done
+ while test -f "/tmp/flood.6069";do sleep 1; done
+ while test -f "/tmp/flood.7079";do sleep 1; done
+ while test -f "/tmp/flood.8089";do sleep 1; done
+ while test -f "/tmp/flood.9099";do sleep 1; done
+
+ echo "Synchronization waiting ..."
+
+ # Wait while out.log is growing ..."
+ OLD_SIZE=0
+ CYCLE=0
+ while : ; do
+ NEW_SIZE=$(stat --printf=%s out.log)
+
+ test $((NEW_SIZE - OLD_SIZE)) -gt 0 || break
+
+ OLD_SIZE=$NEW_SIZE
+
+ CYCLE=$((CYCLE + 1))
+ test $CYCLE -gt 100 && break
+
+ sleep 2
+ done
+
+ kill $PID_OF_WATCH
+
+ echo "Checking results"
+ #
+ # $ ls -l $WATCHED_DIR
+ # total 0
+ # $
+ #
+ rlAssertEquals "The watched directory is empty" "_1" "_$(ls -l $WATCHED_DIR | wc -l)"
+ rlPhaseEnd
+
+ rlPhaseStartCleanup
+ rm -rf $WATCHED_DIR
+ rlPhaseEnd
+ rlJournalPrintText
+rlJournalEnd

[Denys Vlasenko]

Post by Jakub Filak
Related to #657

Pushed patches 3-5. thanks!

[Denys Vlasenko]

Post by Jakub Filak
Related to #657

Pushed, thanks.